<?xml version="1.0"?>
<!DOCTYPE sun SYSTEM "docs.dtd" [

  <!-- Define some character constants. -->
  <!ENTITY column.id.char '$'>
  <!ENTITY indirection.char '@'>
  <!ENTITY ucd.prefix 'ucd$'>
  <!ENTITY utype.prefix 'utype$'>
  <!ENTITY param.prefix 'param$'>

  <!-- Define automatically generated text entities. -->
  <!ENTITY jel.func.docs SYSTEM "jel-javadocs.xml">

  <!-- Define external URLs. -->
  <!ENTITY URL.J2SE_DOCS "https://docs.oracle.com/javase/8/docs/">
  <!ENTITY URL.STIL_JAVADOCS
           "http://www.starlink.ac.uk/stil/javadocs/uk/ac/starlink/table/">
  <!ENTITY URL.HEALPIX_FITS
          "https://healpix.sourceforge.io/data/examples/healpix_fits_specs.pdf">

  <!-- Define javadoc base URLs. -->
  <!ENTITY corejavadocs "http://docs.oracle.com/javase/8/docs/api/">
  <!ENTITY stiljavadocs "http://www.starlink.ac.uk/stil/javadocs/">
  <!ENTITY mbtwww "https://www.star.bristol.ac.uk/mbt/">
  <!ENTITY stiltsjavadocs "&mbtwww;starjavadocs/">
  <!ENTITY starjavasrc "https://github.com/Starlink/starjava/blob/master/">

  <!-- Short descriptions for each command. -->
  <!ENTITY % purpose-defs SYSTEM "purpose-defs.xml">
  %purpose-defs;

  <!-- Description of table schemes. -->
  <!ENTITY scheme-docs SYSTEM 'scheme-docs.xml'>

  <!-- Description of processing filters. -->
  <!ENTITY filter-docs SYSTEM 'filter-docs.xml'>

  <!-- Description of output modes. -->
  <!ENTITY mode-docs SYSTEM 'mode-docs.xml'>

  <!-- Description of graphical output modes. -->
  <!ENTITY paintmode-docs SYSTEM 'paintmode-docs.xml'>

  <!-- Description of plot2 objects. -->
  <!ENTITY layertype-docs SYSTEM 'layertype-docs.xml'>
  <!ENTITY shapemode-docs SYSTEM 'shapemode-docs.xml'>

  <!-- Description of I/O handlers. -->
  <!ENTITY in-ascii SYSTEM 'in-ascii.xml'>
  <!ENTITY in-cdf SYSTEM 'in-cdf.xml'>
  <!ENTITY in-colfits SYSTEM 'in-colfits-basic.xml'>
  <!ENTITY in-csv SYSTEM 'in-csv.xml'>
  <!ENTITY in-ecsv SYSTEM 'in-ecsv.xml'>
  <!ENTITY in-pds4 SYSTEM 'in-pds4.xml'>
  <!ENTITY in-parquet SYSTEM 'in-parquet.xml'>
  <!ENTITY in-hapi SYSTEM 'in-hapi.xml'>
  <!ENTITY in-feather SYSTEM 'in-feather.xml'>
  <!ENTITY in-fits SYSTEM 'in-fits.xml'>
  <!ENTITY in-gbin SYSTEM 'in-gbin.xml'>
  <!ENTITY in-ipac SYSTEM 'in-ipac.xml'>
  <!ENTITY in-mrt SYSTEM 'in-mrt.xml'>
  <!ENTITY in-tst SYSTEM 'in-tst.xml'>
  <!ENTITY in-votable SYSTEM 'in-votable.xml'>
  <!ENTITY in-wdc SYSTEM 'in-wdc.xml'>
  <!ENTITY out-ascii SYSTEM 'out-ascii.xml'>
  <!ENTITY out-csv SYSTEM 'out-csv.xml'>
  <!ENTITY out-ecsv SYSTEM 'out-ecsv.xml'>
  <!ENTITY out-parquet SYSTEM 'out-parquet.xml'>
  <!ENTITY out-feather SYSTEM 'out-feather.xml'>
  <!ENTITY out-fits SYSTEM 'out-fits.xml'>
  <!ENTITY out-html SYSTEM 'out-html.xml'>
  <!ENTITY out-ipac SYSTEM 'out-ipac.xml'>
  <!ENTITY out-latex SYSTEM 'out-latex.xml'>
  <!ENTITY out-mirage SYSTEM 'out-mirage.xml'>
  <!ENTITY out-text SYSTEM 'out-text.xml'>
  <!ENTITY out-tst SYSTEM 'out-tst.xml'>
  <!ENTITY out-votable SYSTEM 'out-votable.xml'>

  <!-- Summary paragraphs containing usage for each commmand. -->
  <!ENTITY arrayjoin-summary SYSTEM 'arrayjoin-summary.xml'>
  <!ENTITY calc-summary SYSTEM 'calc-summary.xml'>
  <!ENTITY cdsskymatch-summary SYSTEM 'cdsskymatch-summary.xml'>
  <!ENTITY cone-summary SYSTEM 'cone-summary.xml'>
  <!ENTITY coneskymatch-summary SYSTEM 'coneskymatch-summary.xml'>
  <!ENTITY datalinklint-summary SYSTEM 'datalinklint-summary.xml'>
  <!ENTITY funcs-summary SYSTEM 'funcs-summary.xml'>
  <!ENTITY pixfoot-summary SYSTEM 'pixfoot-summary.xml'>
  <!ENTITY pixsample-summary SYSTEM 'pixsample-summary.xml'>
  <!ENTITY plot2cube-summary SYSTEM 'plot2cube-summary.xml'>
  <!ENTITY plot2plane-summary SYSTEM 'plot2plane-summary.xml'>
  <!ENTITY plot2sky-summary SYSTEM 'plot2sky-summary.xml'>
  <!ENTITY plot2sphere-summary SYSTEM 'plot2sphere-summary.xml'>
  <!ENTITY plot2corner-summary SYSTEM 'plot2corner-summary.xml'>
  <!ENTITY plot2time-summary SYSTEM 'plot2time-summary.xml'>
  <!ENTITY plot2d-summary SYSTEM 'plot2d-summary.xml'>
  <!ENTITY plot3d-summary SYSTEM 'plot3d-summary.xml'>
  <!ENTITY plothist-summary SYSTEM 'plothist-summary.xml'>
  <!ENTITY regquery-summary SYSTEM 'regquery-summary.xml'>
  <!ENTITY server-summary SYSTEM 'server-summary.xml'>
  <!ENTITY sqlclient-summary SYSTEM 'sqlclient-summary.xml'>
  <!ENTITY sqlskymatch-summary SYSTEM 'sqlskymatch-summary.xml'>
  <!ENTITY sqlupdate-summary SYSTEM 'sqlupdate-summary.xml'>
  <!ENTITY taplint-summary SYSTEM 'taplint-summary.xml'>
  <!ENTITY tapquery-summary SYSTEM 'tapquery-summary.xml'>
  <!ENTITY tapresume-summary SYSTEM 'tapresume-summary.xml'>
  <!ENTITY tapskymatch-summary SYSTEM 'tapskymatch-summary.xml'>
  <!ENTITY tcat-summary SYSTEM 'tcat-summary.xml'>
  <!ENTITY tcatn-summary SYSTEM 'tcatn-summary.xml'>
  <!ENTITY tcopy-summary SYSTEM 'tcopy-summary.xml'>
  <!ENTITY tcube-summary SYSTEM 'tcube-summary.xml'>
  <!ENTITY tgridmap-summary SYSTEM 'tgridmap-summary.xml'>
  <!ENTITY tgroup-summary SYSTEM 'tgroup-summary.xml'>
  <!ENTITY tjoin-summary SYSTEM 'tjoin-summary.xml'>
  <!ENTITY tloop-summary SYSTEM 'tloop-summary.xml'>
  <!ENTITY tmatch1-summary SYSTEM 'tmatch1-summary.xml'>
  <!ENTITY tmatch2-summary SYSTEM 'tmatch2-summary.xml'>
  <!ENTITY tmatchn-summary SYSTEM 'tmatchn-summary.xml'>
  <!ENTITY tmulti-summary SYSTEM 'tmulti-summary.xml'>
  <!ENTITY tmultin-summary SYSTEM 'tmultin-summary.xml'>
  <!ENTITY tpipe-summary SYSTEM 'tpipe-summary.xml'>
  <!ENTITY tskymap-summary SYSTEM 'tskymap-summary.xml'>
  <!ENTITY tskymatch2-summary SYSTEM 'tskymatch2-summary.xml'>
  <!ENTITY votcopy-summary SYSTEM 'votcopy-summary.xml'>
  <!ENTITY votlint-summary SYSTEM 'votlint-summary.xml'>
  <!ENTITY xsdvalidate-summary SYSTEM 'xsdvalidate-summary.xml'>
  
  <!-- Raw usage messages for some commands. -->
  <!ENTITY stilts-usage SYSTEM 'stilts-usage.xml'>
  <!ENTITY tcopy-usage SYSTEM 'tcopy-usage.xml'>
  <!ENTITY tcopy-in-usage SYSTEM 'tcopy-in-usage.xml'>

  <!-- Usage messages for match engines. -->
  <!ENTITY matcher-sky-usage SYSTEM 'matcher-sky-usage.xml'>
  <!ENTITY matcher-skyerr-usage SYSTEM 'matcher-skyerr-usage.xml'>
  <!ENTITY matcher-skyellipse-usage SYSTEM 'matcher-skyellipse-usage.xml'>
  <!ENTITY matcher-sky3d-usage SYSTEM 'matcher-sky3d-usage.xml'>
  <!ENTITY matcher-exact-usage SYSTEM 'matcher-exact-usage.xml'>
  <!ENTITY matcher-1d-usage SYSTEM 'matcher-1d-usage.xml'>
  <!ENTITY matcher-2d-usage SYSTEM 'matcher-2d-usage.xml'>
  <!ENTITY matcher-2d_anis-usage SYSTEM 'matcher-2d_anisotropic-usage.xml'>
  <!ENTITY matcher-2d_cuboid-usage SYSTEM 'matcher-2d_cuboid-usage.xml'>
  <!ENTITY matcher-2d_err-usage SYSTEM 'matcher-2d_err-usage.xml'>
  <!ENTITY matcher-2d_ellipse-usage SYSTEM 'matcher-2d_ellipse-usage.xml'>
  <!ENTITY matcher-sky.1d-usage SYSTEM 'matcher-sky.1d-usage.xml'>

  <!-- Layer type descriptions for plot2 commands. -->
  <!ENTITY plot2plane-layers SYSTEM 'plot2plane-layers.xml'>
  <!ENTITY plot2sky-layers SYSTEM 'plot2sky-layers.xml'>
  <!ENTITY plot2corner-layers SYSTEM 'plot2corner-layers.xml'>
  <!ENTITY plot2cube-layers SYSTEM 'plot2cube-layers.xml'>
  <!ENTITY plot2sphere-layers SYSTEM 'plot2sphere-layers.xml'>
  <!ENTITY plot2time-layers SYSTEM 'plot2time-layers.xml'>

  <!-- Automatically generated plot example text. -->
  <!ENTITY plot2d-examples SYSTEM 'plot2d-examples.xml'>
  <!ENTITY plot3d-examples SYSTEM 'plot3d-examples.xml'>
  <!ENTITY plothist-examples SYSTEM 'plothist-examples.xml'>

  <!-- Miscellaneous text -->
  <!ENTITY ttypeN-param SYSTEM 'ttypeN-param.xml'>
  <!ENTITY plotserv-syntax SYSTEM 'plotserv-syntax.xml'>
  <!ENTITY fits-plus SYSTEM 'fits-plus.xml'>
  <!ENTITY fits-wide SYSTEM 'fits-wide.xml'>

  <!-- Automatically generated plot2 figures. -->
  <!ENTITY figs './'>
  <!ENTITY % plot2-defs SYSTEM 'plot2-figdefs.xml'>
  %plot2-defs;

  <!-- Colour map figures. -->
  <!ENTITY FIG.colormaps-abs 'colormaps-abs.svg'>
  <!ENTITY FIG.colormaps-noabs 'colormaps-noabs.svg'>
]>

<sun>

<docinfo>
<title>STILTS - Starlink Tables Infrastructure Library Tool Set</title>

<authorlist>
<author id="mbt"
        email="m.b.taylor@bristol.ac.uk"
        webpage="&mbtwww;"
        affiliation="University of Bristol"
        >Mark Taylor</author>
</authorlist>

<docnumber>256</docnumber>

<docdate>6 November 2024</docdate>

<contactlist>
<contact>STILTS web page:
         <webref url="http://www.starlink.ac.uk/stilts/"/></contact>
<contact>Author email:
         <webref url="mailto:m.b.taylor@bristol.ac.uk"
                 >m.b.taylor@bristol.ac.uk</webref></contact>
<contact>Mailing list:
         <webref url="mailto:topcat-user@jiscmail.ac.uk"
                 >topcat-user@jiscmail.ac.uk</webref></contact>
</contactlist>

</docinfo>

<!-- ................................................................. -->
<docbody>

<abstract>

<px>STILTS is a set of command-line tools for processing tabular data. 
It has been designed for, but is not restricted to, use on astronomical
data such as source catalogues.  
It contains both generic (format-independent) table processing tools and 
tools for processing VOTable documents.  Facilities offered include
crossmatching,
format conversion, 
format validation, 
column calculation and rearrangement, 
row selection,
sorting,
plotting,
statistical calculations
and metadata display.
Calculations on cell data can be performed using a powerful and 
extensible expression language.
</px>

<px>The package is written in pure Java and based on 
<webref url="http://www.starlink.ac.uk/stil/">STIL</webref>,
the Starlink Tables Infrastructure Library.
This gives it high portability, support for many data formats
(including FITS, VOTable, text-based formats and SQL databases),
extensibility and scalability.  Where possible the tools are
written to accept streamed data so the size of tables which can 
be processed is not limited by available memory.
As well as the tutorial and reference information in this document,
detailed on-line help is available from the tools themselves.
</px>

<px>The STILTS application is available under the GNU General Public License
(GPL) though most parts of the library code may alternatively be used
under the GNU Lesser General Public License (LGPL).
</px>

</abstract>

<sect>
<subhead><title>Introduction</title></subhead>

<p>STILTS provides a number of command-line applications which can
be used for manipulating tabular data.
Conceptually it sits between, and uses many of the same classes as, 
the packages 
<webref url="http://www.starlink.ac.uk/stil/">STIL</webref>,
which is a set of Java APIs providing table-related functionality, and 
<webref url="http://www.starlink.ac.uk/topcat">TOPCAT</webref>,
which is a graphical application providing the user with
an interactive platform for exploring one or more tables.
This document is mostly self-contained - it covers some of the
same ground as the STIL and TOPCAT user documents 
(<docxref doc="sun252"/> and <docxref doc="sun253"/> respectively).
</p>

<p>Currently, this package consists of commands in the following categories:
<dl>
<dt>Generic table manipulation</dt>
<dd><p>
    <ref id="tcopy"><code>tcopy</code></ref>,
    <ref id="tpipe"><code>tpipe</code></ref>,
    <ref id="tmulti"><code>tmulti</code></ref>,
    <ref id="tmultin"><code>tmultin</code></ref>,
    <ref id="tcat"><code>tcat</code></ref>,
    <ref id="tcatn"><code>tcatn</code></ref>,
    <ref id="tloop"><code>tloop</code></ref>,
    <ref id="tjoin"><code>tjoin</code></ref>,
    <ref id="arrayjoin"><code>arrayjoin</code></ref>,
    <ref id="tgridmap"><code>tgridmap</code></ref>,
    <ref id="tgroup"><code>tgroup</code></ref>, and
    <ref id="tcube"><code>tcube</code></ref>
    (see <ref id="pipes" plaintextref="yes"/>).
    </p></dd>
<dt>Crossmatching</dt>
<dd><p>
    <ref id="tmatch1"><code>tmatch1</code></ref>,
    <ref id="tmatch2"><code>tmatch2</code></ref>,
    <ref id="tmatchn"><code>tmatchn</code></ref> and
    <ref id="tskymatch2"><code>tskymatch2</code></ref>
    (see <ref id="match" plaintextref="yes"/>).
    </p></dd>
<dt>Plotting</dt>
<dd><p>
    <ref id="plot2plane"><code>plot2plane</code></ref>,
    <ref id="plot2sky"><code>plot2sky</code></ref>,
    <ref id="plot2cube"><code>plot2cube</code></ref>,
    <ref id="plot2sphere"><code>plot2sphere</code></ref>,
    <ref id="plot2corner"><code>plot2corner</code></ref> and
    <ref id="plot2time"><code>plot2time</code></ref>
    (also deprecated old-style plot commands
    <ref id="plot2d"><code>plot2d</code></ref>,
    <ref id="plot3d"><code>plot3d</code></ref> and
    <ref id="plothist"><code>plothist</code></ref>)
    (see <ref id="plot2" plaintextref="yes"/>).
    </p></dd>
<dt>Sky Pixel Operations</dt>
<dd><p>
    <ref id="tskymap"><code>tskymap</code></ref>,
    <ref id="pixfoot"><code>pixfoot</code></ref> and
    <ref id="pixsample"><code>pixsample</code></ref>.
    </p></dd>
<dt>VOTable</dt>
<dd><p>
    <ref id="votcopy"><code>votcopy</code></ref> and
    <ref id="votlint"><code>votlint</code></ref>.
    </p></dd>
<dt>Virtual Observatory access</dt>
<dd><p>
    <ref id="cdsskymatch"><code>cdsskymatch</code></ref>,
    <ref id="cone"><code>cone</code></ref>,
    <ref id="coneskymatch"><code>coneskymatch</code></ref>,
    <ref id="tapquery"><code>tapquery</code></ref>,
    <ref id="tapresume"><code>tapresume</code></ref>,
    <ref id="tapskymatch"><code>tapskymatch</code></ref>,
    <ref id="taplint"><code>taplint</code></ref>,
    <ref id="datalinklint"><code>datalinklint</code></ref> and
    <ref id="regquery"><code>regquery</code></ref>.
    </p></dd>
<dt>SQL databases</dt>
<dd><p>
    <ref id="sqlclient"><code>sqlclient</code></ref>,
    <ref id="sqlupdate"><code>sqlupdate</code></ref> and
    <ref id="sqlskymatch"><code>sqlskymatch</code></ref>.
    </p></dd>
<dt>Miscellaneous</dt>
<dd><p>
    <ref id="calc" plaintextref="yes"><code>calc</code></ref>,
    <ref id="funcs" plaintextref="yes"><code>funcs</code></ref>,
    <ref id="server" plaintextref="yes"><code>server</code></ref> and
    <ref id="xsdvalidate" plaintextref="yes"><code>xsdvalidate</code></ref>.
    </p></dd>
</dl>
See <ref id="classified"/> for an expanded version of this list.
</p>

<p>There are many ways you might want to use these tools;
here are a few possibilities:
<dl>

<dt>In conjunction with TOPCAT</dt>
<dd><p>you can identify a set of processing steps using TOPCAT's interactive
    graphical facilities, and construct a script using the commands
    provided here which can perform the same steps on many 
    similar tables without further user intervention.
    </p></dd>

<dt>Format conversion</dt>
<dd><p>If you have a separate table processing engine and you want to 
    be able to output the results in a somewhat different form, 
    for instance converting it from FITS to VOTable or from
    TABLEDATA-encoded to BINARY-encoded VOTable, or to perform 
    some more scientifically substantial operation such as 
    changing units or coordinate systems, substituting bad values etc,
    you can pass the results through one of the tools here.
    Since on the whole operation is streaming, such conversion can 
    easily and efficiently be done on the fly.
    </p></dd>

<dt>Server-side operations</dt>
<dd><p>The tools provided here are suitable for use on servers, either
    to generate files as part of a web service (perhaps along the 
    lines of the <label>Format conversion</label> item above)
    or as configurable components in a server-based workflow system.
    The <ref id="server"><code>server</code></ref> command may help,
    but is not required, for use in these situations.
    </p></dd>

<dt>Quick look</dt>
<dd><p>You might want to examine the metadata, or a few rows,
    or a statistical summary of a table 
    without having to load the whole thing into TOPCAT or some other
    table viewer application.
    </p></dd>

</dl>
</p>

</sect>

<sect id="stilts-cmd">
<subhead><title>The <code>stilts</code> command</title></subhead>

<p>All the functions available in this package can be used from
a single command, which is usually referred to in this document 
simply as "<code>stilts</code>".  Depending on how you have installed
the package, you may just type "<code>stilts</code>",
or something like
<verbatim>
   java -jar some/path/stilts.jar
</verbatim>
or
<verbatim>
   java -classpath topcat-full.jar uk.ac.starlink.ttools.Stilts
</verbatim>
or something else - 
this is covered in detail in <ref id="invoke"/>.
</p>

<p>In general, the form of a command is
<verbatim><![CDATA[
   stilts <stilts-flags> <task-name> <task-args>
]]></verbatim>
The forms of the parts of this command are described in the following
subsections, and details of each of the available tasks along with
their arguments are listed in the
<ref id="cmdUsage" plaintextref="yes">command reference</ref> 
at the end of this document.
Some of the commands are highly configurable and have a variety of 
parameters to define their operation.  
In many cases however, it's not complicated to use them.
For instance, to convert the data in a FITS table to VOTable format
you might write:
<verbatim><![CDATA[
   stilts tcopy cat.fits cat.vot
]]></verbatim>
</p>


<subsect id="stilts-flags">
<subhead><title>Stilts flags</title></subhead>

<p>Some flags are common to all the tasks in the STILTS package,
and these are specified after the <code>stilts</code> invocation itself
and before the task name.  They generally have the same effect 
regardless of which task is running.  These generic flags are as
follows:
<dl>
<dt><code>-help</code></dt>
<dd><p>Prints a usage message for the <code>stilts</code> command 
    itself and exits.  The message contains a listing of all the
    known tasks.
    </p></dd>

<dt><code>-version</code></dt>
<dd><p>Prints the STILTS version number and exits.
    </p></dd>

<dt><code>-verbose</code></dt>
<dd><p>Causes more verbose information to be written during operation.
    Specifically, what this does is to boost the logging level by 
    one notch.  It may be specified multiple times to increase verbosity
    further.  The flag <code>+verbose</code> can be used to do the opposite
    (reduce the logging level by one notch).
    </p></dd>

<dt><code>-allowunused</code></dt>
<dd><p>Causes unused parameter settings on the command line to be tolerated.
    Normally, any unused parameters on the command line cause a usage
    message to be output and the command to fail, on the assumption
    that if you've supplied a parameter setting that's not doing anything
    it is probably a mistake and you should be given a chance to correct it.
    But if this flag is set, you just get a warning through the logging
    system about any unused parameters, and the command is executed as
    if they weren't there.
    </p></dd>

<dt><code>-prompt</code></dt>
<dd><p>Most of the STILTS commands have a number of parameters which
    will assume sensible defaults if you do not give them explicit
    values on the command line.  If you use the <code>-prompt</code> flag,
    then you will be prompted for every parameter you have not 
    explicitly specified to give you an opportunity to enter a value
    other than the default.
    </p></dd>

<dt><code>-bench</code></dt>
<dd><p>Outputs the elapsed time taken by the task to standard error on
    successful completion.
    </p></dd>

<dt><code>-debug</code></dt>
<dd><p>Sets up output suitable for debugging.  The most visible consequence
    of this is that if an error occurs then a full stacktrace is output, 
    rather than just a user-friendly report.
    </p></dd>

<dt><code>-batch</code></dt>
<dd><p>Some parameters will prompt you for their values, even if they 
    offer legal defaults.  If you use the <code>-batch</code> flag,
    then you won't be prompted at all.
    </p></dd>

<dt><code>-memory</code></dt>
<dd><p>Encourages the command to use java heap memory for caching
    large amounts of data rather than using temporary disk files.
    The default is to use memory for small tables, and disk for large ones.
    This flag is in most cases equivalent to specifying the system
    property <code>-Dstartable.storage=memory</code>.
    </p></dd>

<dt><code>-disk</code></dt>
<dd><p>Encourages the command to use temporary files on disk for caching
    table data.
    The default is to use memory for small tables, and disk for large ones.
    Using this flag may help if you are running out of memory.
    This flag is in most cases equivalent to specifying the system
    property <code>-Dstartable.storage=disk</code>.
    </p></dd>

<dt><code>-memgui</code></dt>
<dd><p>Displays a graphical window while the command is running which
    summarises used and available heap memory.
    May be useful for profiling or understanding resource constraints.
    </p></dd>

<dt><code>-checkversion &lt;vers&gt;</code></dt>
<dd><p>Requires that the version is exactly as given by the string
    <code>&lt;vers&gt;</code>.  If it is not, STILTS will exit with
    an error.  This can be useful when executing in certain controlled
    environments to ensure that the correct version of the application
    is being picked up.
    </p></dd>

<dt><code>-stdout &lt;file&gt;</code></dt>
<dd><p>Sends all normal output from the run to the given file.
    By default this goes to the standard output stream.
    Supplying an empty string or "<code>-</code>" for <code>&lt;file&gt;</code>
    will restore this default behaviour.
    </p></dd>

<dt><code>-stderr &lt;file&gt;</code></dt>
<dd><p>Sends all error output from the run to the given file.
    By default this goes to the standard error stream.
    Supplying an empty string or "<code>-</code>" for <code>&lt;file&gt;</code>
    will restore this default behaviour.
    </p></dd>

</dl>
</p>

<p>If you are submitting an error report, please include the result of
running <code>stilts -version</code> and the output of the troublesome
command with the <code>-debug</code> flag specified.
</p>

</subsect>

<subsect id="task-name">
<subhead><title>Task Names</title></subhead>

<p>The <code>&lt;task-name&gt;</code> part of the command line is the
name of one of the tasks listed in <ref id="cmdUsage"/> - currently 
the available tasks are:
<ul>
<li><code>arrayjoin</code></li>
<li><code>calc</code></li>
<li><code>cdsskymatch</code></li>
<li><code>cone</code></li>
<li><code>coneskymatch</code></li>
<li><code>datalinklint</code></li>
<li><code>funcs</code></li>
<li><code>pixfoot</code></li>
<li><code>pixsample</code></li>
<li><code>plot2corner</code></li>
<li><code>plot2cube</code></li>
<li><code>plot2plane</code></li>
<li><code>plot2sphere</code></li>
<li><code>plot2sky</code></li>
<li><code>plot2time</code></li>
<li><code>plot2d</code></li>
<li><code>plot3d</code></li>
<li><code>plothist</code></li>
<li><code>regquery</code></li>
<li><code>server</code></li>
<li><code>sqlclient</code></li>
<li><code>sqlskymatch</code></li>
<li><code>sqlupdate</code></li>
<li><code>taplint</code></li>
<li><code>tapquery</code></li>
<li><code>tapresume</code></li>
<li><code>tapskymatch</code></li>
<li><code>tcat</code></li>
<li><code>tcatn</code></li>
<li><code>tcopy</code></li>
<li><code>tcube</code></li>
<li><code>tgridmap</code></li>
<li><code>tgroup</code></li>
<li><code>tjoin</code></li>
<li><code>tloop</code></li>
<li><code>tmatch1</code></li>
<li><code>tmatch2</code></li>
<li><code>tmatchn</code></li>
<li><code>tmulti</code></li>
<li><code>tmultin</code></li>
<li><code>tpipe</code></li>
<li><code>tskymap</code></li>
<li><code>tskymatch2</code></li>
<li><code>votcopy</code></li>
<li><code>votlint</code></li>
<li><code>xsdvalidate</code></li>
</ul>
</p>

</subsect>

<subsect id="task-args">
<subhead><title>Task Arguments</title></subhead>

<p>The <code>&lt;task-args&gt;</code> part of the command line is a
list of parameter assignments, 
each giving the value of one of the named parameters belonging to 
the task which is specified in the <code>&lt;task-name&gt;</code> part.
</p>

<p>The general form of each parameter assignment is
<verbatim><![CDATA[
   <param-name>=<param-value>
]]></verbatim>
If you want to set the parameter to the null value, which is legal for
some but not all parameters, use the special string "<code>null</code>",
or just leave the value blank ("<code>&lt;param-name&gt;=</code>").
In some cases you can optionally leave out the <code>&lt;param-name&gt;</code>
part of the assignment (i.e. the parameter is positionally determined); 
this is indicated in the task's usage description if the parameter
is described like <code>[&lt;param-name&gt;=]&lt;param-value&gt;</code>
rather than <code>&lt;param-name&gt;=&lt;param-value&gt;</code>.
If the <code>&lt;param-value&gt;</code> contains spaces or other special
characters, then in most cases, such as from the Unix shell, you will
have to quote it somehow.  How this is done depends on your platform,
but usually surrounding the whole value in single quotes will do the trick.
</p>

<p>Tasks may have many parameters, and you don't have to set all of
them explicitly on the comand line.  For a parameter which you don't
set, two things can happen.  In many cases, it will default to some
sensible value.  Sometimes however, you may be prompted for the value
to use.
In the latter case, a line like this will be written to the terminal:
<verbatim>
   matcher - Name of matching algorithm [sky]:
</verbatim>
This is prompting you for the value of the parameter named 
<code>matcher</code>.  "Name of matching algorithm" is a short 
description of what that parameter does.  "<code>sky</code>" is
the default value (if there is no default, no value will appear
in square brackets).
At this point you can do one of four things:
<ul>
<li>Hit return - this will select the default value if there is one.
    If there is no default, this is equivalent to entering 
    "<code>null</code>".</li>
<li>Enter a value for the parameter explicitly.
    The special value "<code>null</code>" means the null value,
    which is legal for some, but not all parameters.
    If the value you enter is not legal, you will see an error 
    message and you will be invited to try again. </li>
<li>Enter "<code>help</code>" or a question mark "<code>?</code>".
    This will output a message
    giving a detailed description of the parameter
    and prompt you again.</li>
<li>Bail out by hitting ctrl-C or whatever is usual on your platform.</li>
</ul>
Under normal circumstances, most parameters which have a legal default
value will default to it if they are not set on the command line,
and you will only be prompted for those where there is no default or
the program thinks there's a good chance you might not want to use it.
You can influence this however using flags to the <code>stilts</code>
command itself (see <ref id="stilts-flags"/>). 
If you supply the <code>-prompt</code> flag, then you will be prompted
for every parameter you have not explicitly set.  If you supply
<code>-batch</code> on the other hand, you won't be prompted for 
any parameters (and if you fail to set any without legal default
values, the task will fail).
</p>

<p>If you want to see the actual values of the parameters for a task
as it runs,
including prompted values and defaulted ones 
which you haven't specified explicitly,
you can use the <code>-verbose</code> flag after the <code>stilts</code>
command:
<verbatim><![CDATA[
   % stilts -verbose tcopy cat.fits cat.vot ifmt=fits
   INFO: tcopy in=cat.fits out=cat.vot ifmt=fits ofmt=(auto)
]]></verbatim>
</p>

<p>If you make a parameter assignment on the command line for a
parameter which is not used by the task in question, STILTS
will issue an error message and the task will fail.
Note some parameters are only used dependent on the presence or
values of other parameters, so even supplying a parameter which is
documented in the task's usage can have this effect.
This is done on the assumption that if you have supplied a spurious
parameter it's probably a mistake and you should be given the
opportunity to correct it.
But if you want to be free to make these mistakes without the
task failing, you can supply the <code>-allowunused</code> flag
as described in <ref id="stilts-flags"/>, in which case they will
just result in a warning.
</p>

<p>Note that when running STILTS from the shell,
it may be necessary to quote some parameter values,
in case they contain spaces or other characters which the shell
may try to interpret.
This can typically be done by writing assignments of the form
<verbatim>
   &lt;param-name&gt;='&lt;param-value&gt;'
</verbatim>
but things can get more hairy; see <ref id="quoting"/> for more detail.
</p>

<p>Extensive help is available from <code>stilts</code> 
itself about task and its parameters, as described in the next section.
</p>

</subsect>

<subsect>
<subhead><title>Getting Help</title></subhead>

<p>As well as the command descriptions in this document
(especially the reference section <ref id="cmdUsage"/>)
you can get help for STILTS usage from the command itself.
Typing
<verbatim>
   stilts -help
</verbatim>
results in this output:
<verbatim>&stilts-usage;</verbatim>
</p>

<p>For help on the individual tasks, including their parameter lists,
you can supply the word <code>help</code>
after the task name, so for instance
<verbatim>
   stilts tcopy help
</verbatim>
results in
<verbatim>&tcopy-usage;</verbatim>
</p>

<p>Finally, you can get help on any of the parameters of a task
by writing <code>help=&lt;param-name&gt;</code>, like this:
<verbatim>
   stilts tcopy help=in
</verbatim>
gives
<verbatim>&tcopy-in-usage;</verbatim>
If you use "<code>*</code>" instead of a parameter name in this usage, 
help for all the parameters will be printed.  Note that in most shells
you will probably need to quote the asterisk, so you should write
<verbatim>
   stilts tcopy help='*'
</verbatim>
</p>

<p>In some cases, as described in <ref id="task-args"/>, you will
be prompted for the value of a parameter with a line something like this:
<verbatim>
   matcher - Name of matching algorithm [sky]:
</verbatim>
In this case, if you enter "<code>help</code>" or a question mark,
then the parameter help entry will be printed to the screen, and 
the prompt will be repeated.
</p>

<p>For more detailed descriptions of the tasks, which includes
explanatory comments and examples as well as the information above,
see the full task descriptions in the
<ref id="cmdUsage" plaintextref="yes">Command Reference</ref>.
</p>

</subsect>

</sect>


<sect id="invoke">
<subhead><title>Invocation</title></subhead>

<p>There are a number of ways of invoking commands
in the <code>stilts</code> application,
depending on how you have installed the package.
This section describes how to invoke it from the command line.
Other options are using it from Jython (the Java implementation of
the Python language) as described in <ref id="jystilts"/>,
invoking it over HTTP as described in <ref id="serverMode"/>,
and invoking it from within a Java application
as described in <ref id="taskApi"/>.
</p>

<p>If you're using a Unix-like operating system,
the easiest way is to use the <code>stilts</code> script.
It is a simple shell script which just invokes java with the
right classpath and the supplied arguments.
</p>

<p>If you have a full starjava installation the <code>stilts</code> script
is in the <code>starjava/bin</code> directory.
Otherwise you can download it separately from wherever you got your
STILTS installation in the first place, or find it 
at the top of the <code>stilts.jar</code> or <code>topcat-*.jar</code> file
that contains your STILTS installation, so do something like
<verbatim>
   unzip stilts.jar stilts
   chmod +x stilts
</verbatim>
to extract it (if you don't have <code>unzip</code>, 
try <code>jar xvf stilts.jar stilts</code>).
If you have mounted the <code>topcat-all.dmg</code> file on MacOS
(<code>hdiutil attach topcat-all.dmg</code>) it will probably be
present at a location like
<code>/Volumes/topcat/TOPCAT.app/Contents/Resources/app/stilts</code>.
</p>

<p>To run using the <code>stilts</code> script, first make sure that
both the <code>java</code> 
executable and the <code>stilts</code> script itself are on your path,
and that the <code>stilts.jar</code> or <code>topcat-*.jar</code>
jar file is in the same directory as <code>stilts</code>.
Then the form of invocation is:
<verbatim><![CDATA[
   stilts <java-flags> <stilts-flags> <task-name> <task-args>
]]></verbatim>
A simple example would be:
<verbatim>
   stilts votcopy format=binary t1.xml t2.xml
</verbatim>
in this case, as often, there are no <code>&lt;java-flags&gt;</code> or
<code>&lt;stilts-flags&gt;</code>.
If you use the <code>-classpath</code>
argument or have a CLASSPATH environment variable set, 
then classpath elements thus specified will be added to the classpath
required to run the command.
The examples in the 
command descriptions below use this form for convenience.
</p>

<p>If you don't have a Unix-like shell available however,
you will need to invoke
Java directly with the appropriate classes on your classpath.
If you have the file <code>stilts.jar</code>, in most cases you can 
just write:
<verbatim><![CDATA[
   java <java-flags> -jar stilts.jar <stilts-flags> <task-name> <task-args>
]]></verbatim>
which in practice would look something like
<verbatim>
   java -jar /some/where/stilts.jar votcopy format=binary t1.xml t2.xml
</verbatim>
</p>

<p>In the most general case, Java's <code>-jar</code> flag might be
no good, for one of the following reasons:
<ol>
<li>You have the classes in some form other than the <code>stilts.jar</code>
    file (such as <code>topcat-full.jar</code>)</li>
<li>You need to specify some extra classes on the classpath, which is
    required e.g. for use with 
    <ref id="jdbcConfig" plaintextref="yes">JDBC</ref> 
    or if you are 
    <ref id="jelExtend" plaintextref="yes">extending the commands</ref>
    using your own classes at runtime</li>
</ol>
In this case, you will need an invocation of this form:
<verbatim><![CDATA[
   java <java-flags> -classpath <class-path> 
        uk.ac.starlink.ttools.Stilts <stilts-flags> <task-name> <task-args>
]]></verbatim>
The example above in this case would look something like:
<verbatim>
   java -classpath /some/where/topcat-full.jar uk.ac.starlink.ttools.Stilts 
        votcopy format=binary t1.xml t2.xml
</verbatim>
</p>

<p>Finally, as a convenience, it is possible to run STILTS from a
<webref url="http://www.starlink.ac.uk/topcat/">TOPCAT</webref> 
installation by using its <code>-stilts</code> flag, like this:
<verbatim><![CDATA[
   topcat <java-flags> -stilts <stilts-flags> <task-name> <task-args>
]]></verbatim>
This is possible because TOPCAT is built on top of STILTS, so contains
a superset of its code.
</p>

<p>The 
<code>&lt;stilts-flags&gt;</code>,
<code>&lt;task-name&gt;</code> and
<code>&lt;task-args&gt;</code> 
parts of these invocations are explained in <ref id="stilts-cmd"/>,
and the 
<code>&lt;class-path&gt;</code> and
<code>&lt;java-flags&gt;</code>
parts are explained in the following subsections.
</p>

<subsect id="jvmClasspath">
<subhead><title>Class Path</title></subhead>

<p>The classpath is the list of places that Java looks to find
the bits of compiled code that it uses to run an application.
Depending on how you have done your installation the core STILTS
classes could be in various places, but they are probably in a
file with one of the names 
<code>stilts.jar</code>,
<code>topcat-full.jar</code> or
<code>topcat-extra.jar</code>.
The full pathname of one of these files can therefore be used as
your classpath.  In some cases these files are self-contained and
in some cases they reference other jar files in the filesystem -
this means that they may or may not continue to work if you 
move them from their original location.
</p>

<p>Under certain circumstances the tools might need additional classes,
for instance:
<ul>
<li>JDBC drivers (see <ref id="jdbcConfig"/>)</li>
<li>Providing extended algebraic functions 
    (see <ref id="jelExtend"/>)</li>
<li>Installing I/O handlers for new table formats
    (see <docxref doc="sun252" loc="pluggableIO"/>)</li>
</ul>
In this case the classpath must contain a list of all the jar files in which
the required classes can be found, separated by colons (unix) or
semicolons (MS Windows).  Note that even if all your jar files
are in a single directory you can't use the name of
that directory as a class path - you must name each jar file,
separated by colons/semicolons.
</p>

</subsect>

<subsect id="jvmArgs">
<subhead><title>Java Flags</title></subhead>

<p>In most cases it is not necessary to specify any additional 
arguments to the Java runtime, but it can be useful in certain
circumstances.  The two main kinds of options you might want to
specify directly to Java are these:

<dl>

<dt>System properties</dt>
<dd><p>System properties are a way of getting information into the
    Java runtime from the outside, rather like environment variables.
    There is a list of the ones which have significance to STILTS 
    in <ref id="sysProperties"/>.  You can set them from the 
    command line using a flag of the form <code>-Dname=value</code>.
    So for instance to ensure that temporary files are written to
    the <code>/home/scratch</code> directory, you could use the flag
    <verbatim>
   -Djava.io.tmpdir=/home/scratch
    </verbatim>
    </p></dd>

<dt>Memory size</dt>
<dd><p>Java runs with a fixed amount of 'heap' memory; this is 
    typically 64Mb by default.  
    If one of the tools fails with a message that says it's out of memory
    then this has proved too small for the job in hand.  You can increase the 
    heap memory with the <code>-Xmx</code> flag.  To set the heap 
    memory size to 256 megabytes, use the flag
    <verbatim>
   -Xmx256M
    </verbatim>
    (don't forget the 'M' for megabyte).  You will probably find
    performance is dreadful if you specify a heap size larger than 
    the physical memory of the machine you're running on.
    </p></dd>

</dl>
</p>

<p>You can specify other options to Java such as tuning and profiling
flags etc, but if you want to do that sort of thing 
you probably don't need me to tell you about it.
</p>

<p>If you are using the <code>stilts</code> command-line script,
any flags to it starting <code>-D</code> or <code>-X</code> are passed
directly to the <code>java</code> executable.  
You can pass other flags to Java with the <code>stilts</code> script's
<code>-J</code> flag; for instance:
<verbatim>
   stilts -Xmx4M -J-verbose:gc calc 'mjdToIso(0)'
</verbatim>
is equivalent to
<verbatim>
   java -Xmx4M -verbose:gc -jar stilts.jar calc 'mjdToIso(0)'
</verbatim>
</p>

</subsect>


<subsect id="sysProperties">
<subhead><title>System Properties</title></subhead>

<p>System properties are a way of getting information into the
Java runtime - they are a bit like environment variables.
There are two ways to set them when using STILTS: either
on the command line using arguments of the form
<code>-Dname=value</code> (see <ref id="jvmArgs"/>)
or in a file in your home directory named 
<code>.starjava.properties</code>, in the form of a
<code>name=value</code> line.
Thus submitting the flag
<verbatim>
   -Dvotable.strict=false
</verbatim>
on the command line is equivalent to having the following in your
<code>.starjava.properties</code> file:
<verbatim>
   #  Force strict interpretation of the VOTable standard.
   votable.strict=false
</verbatim>
</p>

<p>The following system properties have special significance to STILTS:
<dl>

<dt><code>http.proxyHost</code></dt>
<dd><p>Can be used to force HTTP access to go via a named proxy;
    may be required if you are attempting access to remote data or services
    from behind a firewall configured to block direct HTTP connections.
    See java documentation for this property for more details.
    </p></dd>

<dt><code>java.awt.headless</code></dt>
<dd><p>May need to be set to "<code>true</code>" if running the 
    plotting tasks on a headless server.  
    You only need to worry about this if you see error messages
    complaining about headlessness.
    </p></dd>

<dt><code>java.io.tmpdir</code></dt>
<dd><p>The directory in which STILTS will write any temporary files it needs.
    This defaults to the system temporary directory
    (e.g. <code>/tmp</code> on Unix),
    so if working with large unmapped (e.g. CSV) tables
    on a machine with limited space on the default disk,
    it may be necessary to change it.
    </p></dd>

<dt><code>java.util.concurrent.ForkJoinPool.common.parallelism</code></dt>
<dd><p>Controls the level of parallelisation done by certain processing,
    currently mainly visualisation.  By default it is typically set to
    one less than the number of processing cores on the current machine.
    To inhibit parallelisation (e.g. if you suspect that the parallel
    output is giving different results to sequential processing)
    you can set this to 1.
    </p></dd>

<dt><code>jdbc.drivers</code></dt>
<dd><p>Can be set to a (colon-separated) list of JDBC driver classes
    using which SQL databases can be accessed
    (see <ref id="jdbcConfig"/>).
    </p></dd>

<dt><code>jel.classes</code></dt>
<dd><p>Can be set to a (colon-separated) list of classes containing
    static methods which define user-provided
    functions for synthetic columns or subsets.
    (see <ref id="jelExtend"/>).
    </p></dd>

<dt><code>mark.workaround</code></dt>
<dd><p>If set to "true", this will work around a bug in the 
    <code>mark()</code>/<code>reset()</code> methods of some java
    <code>InputStream</code> classes.  These are rather common,
    including in Sun's J2SE system libraries.
    Use this if you are seeing errors that say something like
    "<code>Resetting to invalid mark</code>".
    Currently defaults to "false".</p></dd>

<dt><code>service.maxparallel</code></dt>
<dd><p>Raises the maximum number of concurrent queries that may be made
    during a multi-cone operation.
    You should only increase this value <strong>with great care</strong>
    since you risk overloading servers and becoming unpopular with
    data centres.
    As a rule, you should only increase this value if you have
    obtained permission from the data centres whose services
    on which you will be using the increased parallelism.
    </p></dd>

<dt><code>auth.username</code></dt>
<dt><code>auth.password</code></dt>
<dd><p>If these are both set, they will provide username and password
    for accessing authenticated resources.
    Any time the application is refused access to an HTTP connection
    and knows how to try to authenticate,
    it will try again using these credentials.
    In each case the values may be
    either the username/password itself,
    or of the form "<code>@&lt;filename&gt;</code>",
    in which case the value is read from the first line of the named file.
    This replaces the normal behaviour of asking for a username and
    password on the console; see the
    <ref id="AuthManager" plaintextref="yes">section on Authentication</ref>
    for more details.
    Since this setting will pass the username and password information to any
    protected resource without checking it is the intended destination,
    this can potentially leak secret information to third parties,
    so these properties should be set with care.
    </p></dd>

<dt><code>auth.schemes</code></dt>
<dd><p>Configures the list of authentication schemes
    that will be considered when connecting to services issuing
    a WWW-Authenticate challenge.
    A comma-separated list of scheme names or
    <javadoc docset="&stiltsjavadocs;"
             class="uk.ac.starlink.auth.AuthScheme"/>
    implementation classnames may be provided.
    </p></dd>

<dt><code>startable.readers</code></dt>
<dd><p>Can be set to a (colon-separated) list of custom table format input
    handler classes (see <docxref doc="sun252" loc="pluggableIO"/>).
    Each class must implement the
    <code>uk.ac.starlink.table.TableBuilder</code> interface,
    and must have a no-arg constructor.
    The readers thus named will be available
    alongside the standard ones listed in <ref id="inFormats"/>.
    </p></dd>

<dt><code>startable.schemes</code></dt>
<dd><p>Can be set to a (colon-separated) list of custom table scheme handler
    classes.  Each class must implement the
    <code>uk.ac.starlink.table.TableScheme</code> interface,
    and must have a no-arg constructor.
    The schemes thus named will be available
    alongside the standard ones listed in <ref id="TableScheme"/>.
    </p></dd>

<dt><code>startable.storage</code></dt>
<dd><p>Can be set to determine the default storage policy.
    Setting it to "<code>disk</code>" has basically the same effect as
    supplying the "<code>-disk</code>" argument on the command line
    (see <ref id="stilts-flags"/>).
    Other possible values are "<code>adaptive</code>", "<code>memory</code>",
    "<code>sideways</code>" and "<code>discard</code>";
    see <docxref doc="sun252" loc="storagePolicy"/>.
    The default is "<code>adaptive</code>", which means storing smaller
    tables in memory, and larger ones on disk.
    </p></dd>

<dt><code>startable.unmap</code></dt>
<dd><p>Determines whether and how unmapping of memory mapped buffers is done.
    Possible values are "<code>sun</code>" (the default), 
    "<code>cleaner</code>", "<code>unsafe</code>" or "<code>none</code>".
    In most cases you are advised to leave this alone, but in the event of
    unmapping-related JVM crashes (not expected!), setting it to
    <code>none</code> may help.
    </p></dd>

<dt><code>startable.writers</code></dt>
<dd><p>Can be set to a (colon-separated) list of custom table format output
    handler classes (see <docxref doc="sun252" loc="pluggableIO"/>).
    Each class must implement the
    <code>uk.ac.starlink.table.StarTableWriter</code> interface,
    and must have a no-arg constructor.
    The writers thus named will be available
    alongside the standard ones listed in <ref id="outFormats"/>.
    </p></dd>

<dt><code>votable.namespacing</code></dt>
<dd><p>Determines how namespacing is handled in input VOTable documents.
    Known values are 
    "<code>none</code>" (no namespacing, xmlns declarations
    in VOTable document will probably confuse parser),
    "<code>lax</code>" (anything that looks like it is probably a VOTable
    element will be treated as a VOTable element) and
    "<code>strict</code>" (VOTable elements must be properly declared in one
    of the correct VOTable namespaces).
    May also be set to the classname of a
    <code>uk.ac.starlink.votable.Namespacing</code> implementation.
    The default is "<code>lax</code>".
    </p></dd>

<dt><code>votable.strict</code></dt>
<dd><p>Controls the behaviour when encountering a VOTable
    <code>FIELD</code> or <code>PARAM</code> element with
    a <code>datatype</code> attribute of
    <code>char</code>/<code>unicodeChar</code>,
    and no <code>arraysize</code> attribute.
    The VOTable standard says this indicates a single character,
    but some VOTables omit arraysize specification by accident when
    they intend <code>arraysize="*"</code>.
    If <code>votable.strict</code> is set <code>true</code>,
    a missing arraysize will be interpreted as meaning a single character,
    and if <code>false</code>, it will be interpreted as a variable-length
    array of characters (a string).
    The default is <code>true</code>.
    </p></dd>

<dt><code>votable.version</code></dt>
<dd><p>Selects the version of the VOTable standard which output VOTables
    will conform to by default.
    May take the values "<code>1.0</code>", "<code>1.1</code>",
    "<code>1.2</code>", "<code>1.3</code>" or "<code>1.4</code>".
    By default, version 1.4 VOTables are written.
    </p></dd>

</dl>
</p>

</subsect>

<subsect id="jdbcConfig">
<subhead><title>JDBC Configuration</title></subhead>

<p>This section describes additional configuration which must be
done to allow the commands to access SQL-compatible relational databases
for reading  or writing tables.
If you don't need to talk to SQL-type databases,
you can ignore the rest of this section.
The steps described here are the standard ones
for configuring JDBC (which sort-of stands for Java Database Connectivity);
you can find more information on that on the web.
The best place to look may be within the documentation 
of the RDBMS you are using.
</p>

<p>To use STILTS with SQL-compatible databases you must:
<ul>
<li>Have access to an SQL-compatible database locally or over the network</li>
<li>Have a JDBC driver appropriate for that database</li>
<li>Install that driver for use with STILTS</li>
<li>Know the format the driver uses for URLs to access database tables</li>
<li>Have appropriate privileges on the database to perform the
    desired operations</li>
</ul>
Installing the driver consists of two steps:
<ol>
<li>Ensure that the classpath you are using includes this driver class
    as described in <ref id="jvmClasspath"/></li>
<li>Set the <code>jdbc.drivers</code> system property to the name of the
    driver class as described in <ref id="sysProperties"/></li>
</ol>
</p>

<p>Here is an example of using <code><ref id="tpipe">tpipe</ref></code>
to write the results
of an SQL query on a table in a MySQL database as a VOTable:
<verbatim><![CDATA[
   stilts -classpath /usr/local/jars/mysql-connector-java.jar \
          -Djdbc.drivers=com.mysql.jdbc.Driver \
          tpipe \
          in="jdbc:mysql://localhost/db1#SELECT id, ra, dec FROM gsc WHERE mag < 9" \
          ofmt=votable gsc.vot
]]></verbatim>
or invoking Java directly:
<verbatim><![CDATA[
   java -classpath stilts.jar:/usr/local/jars/mysql-connect-java.jar \
        -Djdbc.drivers=com.mysql.jdbc.Driver \
        uk.ac.starlink.ttools.Stilts tpipe \
        in="jdbc:mysql://localhost/db1#SELECT id, ra, dec FROM gsc WHERE mag < 9" \
        ofmt=votable out=gsc.vot
]]></verbatim>
You have to exercise some care to get the arguments
in the right order here - see <ref id="invoke"/>.
</p>

<p>Alternatively, you can set some of this up beforehand to make the
invocation easier.  If you set your CLASSPATH environment variable
to include the driver jar file (and the STILTS classes if you're 
invoking Java directly rather than using the scripts), and if you
put the line
<verbatim>
   jdbc.drivers=com.mysql.jdbc.Driver
</verbatim>
in the <code>.starjava.properties</code> file in your home directory,
then you could avoid having to give the <code>-classpath</code> and 
<code>-Djdbc.drivers</code> flags respectively.
</p>

<p>Below are presented the results of some experiments with JDBC drivers.
Note however that
<strong>this information may be be incomplete and out of date</strong>.
If you have updates, feel free to pass them on and they may be
incorporated here. 
</p>

<p>To the author's knowledge, STILTS has successfully been used
with the following RDBMSs and corresponding JDBC drivers:
<dl>

<dt><webref url="http://www.mysql.com/">MySQL</webref></dt>
<dd><p>MySQL has been tested on Linux with the
    <webref url="http://www.mysql.com/products/connector/j/"
            >Connector/J</webref> driver and seems to work;
    tested versions are server 3.23.55 with driver 3.0.8 and
    server 4.1.20 with driver 5.0.4.
    Sometimes tables with very many (hundreds of) columns cannot be
    written owing to SQL statement length restrictions.
    Note there is known to be a column metadata bug in version 3.0.6 of the
    driver which can cause a ClassCastException error when tables are written.
    Check the driver's documentation for additional parameters, for instance
    "<code>useUnicode=true&amp;characterEncoding=UTF8</code>" may be required
    to handle some non-ASCII characters.
    </p></dd>

<dt><webref url="http://www.postgresql.org/">PostgreSQL</webref></dt>
<dd><p>PostgreSQL 7.4.1 apparently works with
    <webref url="http://jdbc.postgresql.org/">its own driver</webref>.
    Note the performance of this driver appears to be rather poor,
    at least for writing tables.
    </p></dd>

<dt><webref url="http://www.oracle.com/">Oracle</webref></dt>
<dd><p>You can use Oracle with the JDBC driver that comes as part of its
    <webref url="http://www.oracle.com/technetwork/database/database-technologies/instant-client/"
            >Basic Instant Client Package</webref>.
    </p></dd>

<dt><webref url="http://www.microsoft.com/SQL">SQL Server</webref></dt>
<dd><p>There is more than one JDBC driver known to work with
    SQL Server, including
    <webref url="http://jtds.sourceforge.net/">jTDS</webref>
    and its own JDBC driver.
    Some evidence suggests that jTDS may be the better choice,
    but your mileage may vary.
    </p></dd>

<dt><webref url="https://www.sap.com/uk/products/sybase-ase.html"
            >Sybase ASE</webref></dt>
<dd><p>There has been a successful use of Sybase 12.5.2 and jConnect
    (jconn3.jar) using a JDBC URL like
    "<code>jdbc:sybase:Tds:hostname:port/dbname?user=XXX&amp;password=XXX#SELECT...</code>".
    An earlier attempt using Sybase ASE 11.9.2 failed.
    </p></dd>

</dl>
It is probably possible to use other RDBMSs and drivers,
but you may have to do some homework.
</p>

<p>Here are some example command lines that at least have at some point
got STILTS running with databases:
<dl>

<dt>PostgreSQL</dt>
<dd><p><verbatim>
   stilts -classpath pg73jdbc3.jar \
          -Djdbc.drivers=org.postgresql.Driver ...
</verbatim></p></dd>

<dt>MySQL</dt>
<dd><p><verbatim>
   stilts -classpath mysql-connector-java-3.0.8-bin.jar \
          -Djdbc.drivers=com.mysql.jdbc.Driver ...
</verbatim></p></dd>

<dt>Oracle</dt>
<dd><p><verbatim>
   stilts -classpath ojdbc14.jar \
          -Djdbc.drivers=oracle.jdbc.driver.OracleDriver ...
</verbatim></p></dd>

<dt>SQL Server with jTDS</dt>
<dd><p><verbatim>
   stilts -classpath jtds-1.1.jar \
          -Djdbc.drivers=net.sourceforge.jtds.jdbc.Driver ...
</verbatim></p></dd>

</dl>
</p>

</subsect>

</sect>


<sect id="jystilts">
<subhead><title>JyStilts - STILTS from Python</title></subhead>

<p>Most of the discussions and examples in this document describe
using STILTS as a standalone java application from the command line;
in this case, scripting can be achieved by executing one STILTS
command, followed by another, followed by another, perhaps controlled
from a shell script, with intermediate results stored in files.
</p>

<p>However, it is also possible to invoke STILTS commands from within
the <webref url="http://www.jython.org/">Jython</webref> environment.
Jython is a pure-java implementation of the widely-used
<webref url="http://www.python.org/">Python</webref> scripting language.
Using Jython is almost exactly the same as using the more usual C-based Python,
except that it is not possible to use extensions which use C code.
This means that if you are familiar with Python programming,
it is very easy to string STILTS commands together in Jython.
</p>

<p>This approach has several advantages over the conventional command-line
usage:
<ul>
<li>You can make use of python programming constructions like 
loops, functions and variables</li>
<li>Python syntax can be used to put together parameter values
(especially referencing quoted strings or values containing embedded
spaces) in a way which is often less painful than doing it from
the shell</li>
<li>Intermediate processing stages can be kept in memory (in a python variable)
rather than having to write them out to a file and read them in for the
next command; this can be much more efficient</li>
<li>Because of the previous point, there are separate read, filter,
processing and write commands, which means command lines can be
shorter and less confusing</li>
<li>The java startup overhead (typically a couple of seconds) happens only
once when entering jython, not once for every STILTS command</li>
</ul>
Note however that you will <em>not</em> be able to introduce JyStilts
commands into your larger existing Python programs if those rely on
C-based extensions, such as NumPy and SciPy, since JyStilts will only
run in JPython, while C-based extensions will only run in CPython.
(See however 
<webref url="http://jnumerical.sourceforge.net/index.html"
>JNumeric</webref> for some of the Numpy functionality from Jython.)
</p>

<p>Usage from jython has syntax which is similar to command-line STILTS,
but with a few changes.
The following functions are defined by JyStilts:
<ul>
<li>A function <code>tread</code>, which reads a table from a
file or URL and turns it into a table object in jython
</li>
<li>A table method <code>write</code> which takes a table object and
writes it to file
</li>
<li>A table method for each STILTS <ref id="filterSteps">filter</ref>
(e.g. <code>cmd_head</code>, <code>cmd_select</code>,
  <code>cmd_addcol</code>)
</li>
<li>A table method for each STILTS <ref id="outModes">output mode</ref>
(e.g. <code>mode_out</code>, <code>mode_meta</code>,
  <code>mode_samp</code>),
</li>
<li>A function for each STILTS <ref id="cmdUsage">task</ref> 
(e.g. <code>tmatch2</code>, <code>tcat</code>, <code>plot2sky</code>)
</li>
<li>A number of table methods which make table objects integrate nicely into
the python environment</li>
</ul>
Reasonably detailed documentation for these is provided in the 
usual Python way ("<em>doc strings</em>"), 
and can be accessed using the Python "<code>help</code>" command, however
for full documentation and examples you should refer to this document.
</p>

<p>In JyStilts the input, processing, filtering and output are 
done in separate steps, unlike in command-line STILTS where they all
have to be combined into a single line.  This can make the
flow of execution easier to follow.
A typical sequence will involve:
<ol>
<li>Reading one or more tables from file using
the <code>tread</code> function</li>
<li>Perhaps filtering the input table(s) using one or more of the 
<code>cmd_*</code> filter methods</li>
<li>Performing core processing such as crossmatching</li>
<li>Perhaps filtering the result using one or more of the
<code>cmd_*</code> filter methods</li>
<li>If running interactively, perhaps examining the intermediate results
using one of the <code>mode_*</code> output modes</li>
<li>Writing the final result to a file using the <code>write</code> method</li>
</ol>
</p>

<p>Here is an example command line invocation for crossmatching two tables:
<verbatim>
stilts tskymatch2 in1=survey.fits \
	     icmd1='addskycoords fk4 fk5 RA1950 DEC1950 RA2000 DEC2000' \
	     in2=mycat.csv ifmt2=csv \
	     icmd2='select VMAG>18' \
	     ra1=ALPHA dec1=DELTA ra2=RA2000 dec2=DEC2000 \
	     error=10 join=2not1 \
	     out=matched.fits
</verbatim>
and here is what it might look like in JyStilts:
<verbatim><![CDATA[
>>> import stilts
>>> t1 = stilts.tread('survey.fits')
>>> t1 = t1.cmd_addskycoords(t1, 'fk4', 'fk5', 'RA1950', 'DEC1950', 'RA2000', 'DEC2000')
>>> t2 = stilts.tread('mycat.csv', 'csv')
>>> t2 = t2.cmd_select('VMAG>18')
>>> tm = stilts.tskymatch2(in1=t1, in2=t2, ra1='ALPHA', dec1='DELTA',
...                        error=10, join='2not1')
>>> tm.write('matched.fits')
]]></verbatim>
</p>

<p>When running interactively, it can be convenient to examine the intermediate
results before processing or writing as well, for instance:
<verbatim><![CDATA[
>>> tm.mode_count()
columns: 19   rows: 2102
>>> tm.cmd_keepcols('ID ALPHA DELTA').cmd_head(4).write()
+--------+---------------+-----------+
| ID     | ALPHA         | DELTA     |
+--------+---------------+-----------+
| 262    | 149.82439     | -0.11249  |
| 263    | 150.14438     | -0.11785  |
| 265    | 149.92944     | -0.11667  |
| 273    | 149.93185     | -0.12566  |
+--------+---------------+-----------+
]]></verbatim>
</p>

<p>More detail about how to run JyStilts and its usage 
is given in the following subsections.
</p>

<subsect id="jyrun">
<subhead><title>Running JyStilts</title></subhead>

<p>The easiest way to run JyStilts is to download the standalone
<code>jystilts.jar</code> file from the STILTS web page,
and simply run
<verbatim>
java -jar jystilts.jar   
</verbatim>
This file includes jython itself and all the STILTS and JyStilts classes.
To use the JyStilts commands, you will need to import the stilts
module using a line like "<code>import stilts</code>"
from Jython in the usual Python way.
</p>

<p>Alternatively, you can run JyStilts from an existing Jython installation
using just the <code>stilts.jar</code> file.
First, make sure that Jython is installed;
it is available from <webref url="http://www.jython.org/"/>,
and comes as a self-installing jar file.
JyStilts has been tested, and appears to work, with jython version 2.7.2.
It also works with jython 2.5.* under Java 8 and Java 11,
but jystilts with jython 2.5.* and Java 17 can fail with security problems.
Some earlier versions of JyStilts worked with jython 2.2.1, 
but that no longer seems to be the case; it might be possible
to reinstate this if there is some pressing need.
</p>

<p>To use JyStilts, you then just need to
start jython with the <code>stilts.jar</code> file on your classpath,
for instance like this:
<verbatim>
jython -J-classpath /some/where/stilts.jar
</verbatim>
or (C-shell):
<verbatim>
setenv CLASSPATH /some/where/stilts.jar
jython
</verbatim>
</p>

<p>Optionally, you can extract the <code>stilts.py</code> module 
from the stilts.jar file 
(using a command like "<code>unzip stilts.jar stilts.py</code>")
and put it in a directory on your jython 
<code>sys.path</code> (e.g. <code>jythondir/Lib</code>);
this may cause jython to compile it to bytecode (<code>stilts$py.class</code>)
and thus improve startup time.
Note that in this case you will still need the <code>stilts.jar</code> 
file on your classpath as above.
</p>

</subsect>

<subsect>
<subhead><title>Table I/O</title></subhead>

<p>The <code>tread</code> function reads tables from an external location
into JyStilts.  Its arguments are as follows:
<verbatim>
tread(location, fmt='(auto)', random=False)
</verbatim>
and its return value is a <ref id="jytable">table object</ref>,
which can be interrogated directly, or used in other JyStilts commands.
Usually, the location argument should be a string which gives the
filename or URL at which a table can be found.
You can alternatively use a readable python file (or file-like) object for 
the location, but be aware that this may be less efficient on memory.
As with command-line STILTS, the <code>fmt</code> argument
is one of the options in <ref id="inFormats"/>, but may be
left as the default if the format auto-detectable,
which currently means if the file is in
VOTable, FITS, CDF, ECSV, PDS4, Parquet, Feather or GBIN format.
The <code>random</code> argument can be used to ensure that the returned file
has random (i.e. not sequential-only) access;
for some table formats the default way of reading them in means that
their rows can only be accessed in sequence.
Depending on what processing you are doing, that may or may not be
satisfactory.
</p>

<p>Examples of reading a table are:
<verbatim><![CDATA[
>>> import stilts
>>> t1 = stilts.tread('cat.fits')
>>> t2 = stilts.tread(open('cat.fits', 'rb'))           # less efficient
>>> t3 = stilts.tread('data.csv', fmt='csv', random=True)
]]></verbatim>
</p>

<p>The most straightforward way to write a table
(presumably the result of one or a sequence of JyStilts commands) 
is using the <code>write</code> table method:
<verbatim>
write(self, location=None, fmt='(auto)')
</verbatim>
The <code>location</code> gives either a string which is a filename,
or a writable python file (or file-like) object.
Again, use of a filename is preferred as it may(?) be more efficient.
If no location is supplied, the table will be written to standard output
(useful for inspection, but a bad idea for binary formats or very large tables).
The <code>fmt</code> argument is one of the output formats in 
<ref id="outFormats"/>, but may be left as the default if the
format can be guessed from the filename.
</p>

<p>Examples of writing a table are:
<verbatim><![CDATA[
>>> table.write('out.fits')
>>> table.write(open('out.fits', 'wb'))       #  less efficient?
>>> table.write('catalogue.dat', fmt='csv')
>>> table.write()                             #  display to stdout
]]></verbatim>
</p>

<p>Often it's convenient to combine examining the table with filtering
steps, for instance:
<verbatim><![CDATA[
>>> table.every(100).write()
]]></verbatim>
would write only every hundredth row, and
<verbatim><![CDATA[
>>> (table.cmd_sorthead(10, 'BMAG')
...       .cmd_select('!NULL_VMAG')
...       .cmd_keepcols('BMAG VMAG')
...       .write())
]]></verbatim>
would write only the BMAG and VMAG columns 
for the ten rows in which VMAG is non-null with the lowest BMAG values.
</p>

<p>You can also read and write multiple tables, if you use a table
format for which that is appropriate.
This generally means FITS (which can store tables in multiple extensions)
or VOTable (which can store multiple TABLE elements in one document).
This is done using the <code>treads</code> and <code>twrites</code> functions.
The functions look like this:
<verbatim><![CDATA[
treads(location, fmt='(auto)', random=False)
twrites(tables, location=None, fmt='(auto)')
]]></verbatim>
These are similar to the <code>tread</code> and <code>twrite</code> functions,
except that <code>treads</code> returns a list of tables rather than
a single table, and <code>twrites</code>'s <code>tables</code> argument is
an iterable over tables rather than a single table.
Here is an example of reading multiple tables from a multi-extension FITS
file, counting the rows in each, and then writing them out to a multi-TABLE
VOTable file:
<verbatim><![CDATA[
import stilts
tables = stilts.treads('multi.fits')
print([t.getRowCount() for t in tables])
stilts.twrites(tables, 'multi.vot', fmt='votable')
]]></verbatim>
</p>

</subsect>

<subsect id="jytable">
<subhead><title>Table objects</title></subhead>

<p>The tables read by the <code>tread</code> function and produced
by operating on them within JyStilts have a number of methods
defined on them.
These are explained below.
</p>

<p>First, a number of special methods are defined which allow a table to
behave in python like a sequence of rows:
<dl>
<dt><code>__iter__</code></dt>
<dd><p>This special method means that the table can be treated as an
    <em>iterable</em>, so that for instance
    "<code>for row in table:</code>" will iterate over all rows.
    </p></dd>
<dt><code>__len__</code> <em>(random-access tables only)</em></dt>
<dd><p>This special method means that you can use the expression
    "<code>len(table)</code>" to count the number of rows.
    This method is not available for tables with sequential access only.
    </p></dd>
<dt><code>__getitem__</code> <em>(random-access tables only)</em></dt>
<dd><p>Returns a row at a given index in the table.
    This special method means that you can use indexing expressions like
    "<code>table[3]</code>" or <code>table[0:10]</code> to obtain the
    row or rows corresponding to a given row index or slice.
    This method is not available for tables with sequential access only.
    </p></dd>
<dt><code>__add__</code>, <code>__mul__</code>, <code>__rmul__</code></dt>
<dd><p>These special methods allow the addition and multiplication 
    operators "<code>+</code>" and and "<code>*</code>" to be used with
    the sense of concatenation.
    Thus "<code>table1+table2</code>" will produce a new table with the
    rows of <code>table1</code> followed by the rows of <code>table2</code>.
    Note this will only work if both tables have compatible columns.
    Similarly "<code>table*3</code>" would produce a table like
    <code>table</code> but with all its rows repeated three times.
    </p></dd>
</dl>
In all of these cases, each row object that is accessed is a tuple
of the column values for that row of the table.
The tuple items (table cells) may be accessed using a key which is
a numeric index or slice in the usual way,
or with a key which is a column name, or one of the ColumnInfo objects
returned by <code>columns()</code>.
</p>

<p>Sometimes, the result of a table operation will be a table which
does not have random access.  For such tables you can iterate over
the rows, but not get their row values by indexing.
Non-random-access tables are also peculiar in that <code>getRowCount</code>
returns a negative value.
To take a table which may not have random access and make it capable
of random access, use the <ref id="random"><code>random</code></ref>
filter: "<code>table=table.cmd_random()</code>".
</p>

<p>To a large extent it is possible to duplicate the functions of the
various STILTS commands by writing your own python code based on these
python-friendly table access methods.
Note however that such python-based processing is likely to be
<em>much</em> slower than the STILTS equivalents.
If performance is important to you, you should try in most cases
to use the various <code>cmd_*</code> commands etc for table processing.
</p>

<p>Second, some additional utility methods are defined:
<dl>
<dt><code>count_rows()</code></dt>
<dd><p>Returns the number of rows in the table in the most efficient
    way possible.  If the table is random-access or otherwise knows
    its row count without further calculation, that value is returned.
    Otherwise, the rows are iterated over without reading, which may take
    some time but should be much more efficient than iterating over
    the table as an iterable, since the row cell data itself is not
    retrieved.
    </p></dd>
<dt><code>columns()</code></dt>
<dd><p>Returns a tuple of the column descriptors for the table.
    Each item in the tuple is an instance of the
    <webref url="&URL.STIL_JAVADOCS;ColumnInfo.html">ColumnInfo</webref> class;
    useful methods include <code>getName()</code>,
    <code>getUnitString()</code>, <code>getUCD()</code>.
    <code>str(column)</code> will return its name.
    </p></dd>
<dt><code>coldata(key)</code></dt>
<dd><p>Returns a sequence of the values for the given column.
    The sequence will have the same number of elements as the number of rows
    in the table.  The <code>key</code> argument may be either an integer
    column index (if negative, counts backwards from the end),
    or the column name or info object.
    The returned value will always be iterable (has <code>__iter__</code>),
    but will only be indexable 
    (has <code>__len__</code> and <code>__getitem__</code>) if the table
    is random access.
    </p></dd>
<dt><code>parameters()</code></dt>
<dd><p>Returns a name to value mapping of the table parameters
    (per-table metadata).
    This does not include all the available information about those
    parameters, for instance unit and UCD information is not included.
    For more detailed information, use the <code>StarTable</code> methods.
    Note that as currently implemented, changing the values in the
    returned mapping will not change the actual table parameter values.
    </p></dd>
<dt><code>write(location=None, fmt=None)</code></dt>
<dd><p>Outputs the table.
    The optional <code>location</code> argument gives a filename
    or writable file object,
    and the optional <code>fmt</code> argument gives a format, one of
    the options listed in <ref id="inFormats"/>.
    If <code>location</code> is not supplied, output is to standard output,
    so in an interactive session it will be printed to the terminal.
    If <code>fmt</code> is not supplied, an attempt will be made to guess
    a suitable format based on the location.
    </p></dd>
</dl>
</p>

<p>Third, a set of <code>cmd_*</code> methods corresponding to the
STILTS filters are available;
these are described in <ref id="jyfilter"/>.
</p>

<p>Fourth, a set of <code>mode_*</code> methods corresponding to the
STILTS output modes are available;
these are described in <ref id="jymode"/>.
</p>

<p>Finally, tables are also instances of the 
<webref url="&URL.STIL_JAVADOCS;StarTable.html">StarTable</webref>
interface defined by
<webref url="http://www.starlink.ac.uk/stil/">STIL</webref>,
which is the table I/O layer underlying STILTS.
The full documentation can be found in the user manual and javadocs
on the STIL page, and all the java methods can be used from JyStilts,
but in most cases there are more pythonic equivalents provided,
as described above.
</p>

<p>Here are some examples of these methods in use:
<verbatim><![CDATA[
   >>> import stilts
   >>> xsc = stilts.tread('/data/table/2mass_xsc.xml')  # read table
   >>> xsc.mode_count()                                 # show rows/column count
   columns: 6   rows: 1646844
   >>> print xsc.columns()                              # full info on columns
   (id(String), ra(Double)/degrees, dec(Double)/degrees, jmag(Double)/mag, hmag(Double)/mag, kmag(Double)/mag)
   >>> print [str(col) for col in xsc.columns()]        # column names only
   ['id', 'ra', 'dec', 'jmag', 'hmag', 'kmag']
   >>> row = xsc[1000000]                               # examine millionth row
   >>> print row
   (u'19433000+4003190', 295.875, 40.055286, 14.449, 13.906, 13.374)
   >>> print row[0]                                     # cell by index
   19433000+4003190
   >>> print row['ra'], row['dec']                      # cells by col name
   295.875 40.055286
   >>> print len(xsc)                                   # count rows, maybe slow
   1646844
   >>> print xsc.count_rows()                           # count rows efficiently
   1646844L
   >>> print (xsc+xsc).count_rows()                     # concatenate
   3293688L
   >>> print (xsc*10000).count_rows()
   16468440000L
   >>> for row in xsc:                  # select rows using python commands
   ...     if row[4] - row[3] > 3.0:
   ...         print row[0]
   ... 
   11165243+2925509
   20491597+5119089
   04330238+0858101
   01182715-1013248
   11244075+5218078
   >>>                                  # same thing using stilts (50x faster)
   >>> (xsc.cmd_select('hmag - jmag > 3.0')
   ...     .cmd_keepcols('id')
   ...     .write())
   +------------------+
   | id               |
   +------------------+
   | 11165243+2925509 |
   | 20491597+5119089 |
   | 04330238+0858101 |
   | 01182715-1013248 |
   | 11244075+5218078 |
   +------------------+
]]></verbatim>
</p>

<p>The following are all ways to obtain the value of a given cell
in the table from the previous example.
<verbatim>
    xsc.getCell(99, 0)
    xsc[99][0]
    xsc[99]['id']
    xsc.coldata(0)[99]
    xsc.coldata('id')[99]
</verbatim>
Some of these methods may be more efficient than others.
Note that none of these methods will work if the table has sequential-only
access.
</p>

</subsect>

<subsect id="jyfilter">
<subhead><title>Table filter commands (<code>cmd_*</code>)</title></subhead>

<p>The STILTS table filters documented in <ref id="filterSteps"/> 
are available in JyStilts as table methods
which start with the "<code>cmd_</code>" prefix.
The return value when calling the method on a table object is 
another table object.
The arguments, which are the same as those required for the command-line
version, are supplied as a list of unnamed arguments of the 
<code>cmd_*</code> function.  In general the arguments are strings,
but numbers are accepted where appropriate.
Use the python <code>help</code> command to see the usage of each method.
</p>

<p>So, to use the <ref id="tail"><code>tail</code></ref> filter to
select only the last ten lines of a table, you can write:
<verbatim>
   table.cmd_tail(10)
</verbatim>
To set units of "Hz" for some columns using the 
<ref id="colmeta"><code>colmeta</code></ref> filter write:
<verbatim>
   table.cmd_colmeta('-units', 'Hz', 'AFREQ BFREQ CFREQ')
</verbatim>
Note that where a filter argument is a space-separated list
it must appear as a single argument in the filter invocation,
just as in command-line STILTS.
</p>

<p>The filter commands are also available as module functions.
This means that 
<verbatim>
   stilts.cmd_head(table, 10)
</verbatim>
and
<verbatim>
   table.cmd_head(10)
</verbatim>
have exactly the same meaning.  It's a matter of taste which you prefer.
</p>

</subsect>

<subsect id="jymode">
<subhead><title>Table output modes (<code>mode_*</code>)</title></subhead>

<p>The STILTS table output modes documented in <ref id="outModes"/>
are available in JyStilts as table methods
which start with the "<code>mode_</code>" prefix.
These methods have no return value, but cause something to happen,
in some cases output to be written to standard output.
Some of these methods have named arguments, others have no arguments.
Use the python <code>help</code> command to see the usage of each method.
</p>

<p>These methods are straightforward to use.
The following example calculates statistics for a table and
writes the results to standard output:
<verbatim><![CDATA[
   >>> table.mode_stats()
]]></verbatim>
and this one attempts to send the table via the SAMP communications 
protocol to a running instance of TOPCAT:
<verbatim><![CDATA[
   >>> table.mode_samp(client='topcat')
]]></verbatim>
</p>

<p>The output modes are also available as module functions.
This means that
<verbatim><![CDATA[
   stilts.mode_samp(table, client='topcat')
]]></verbatim>
and
<verbatim><![CDATA[
   table.mode_samp(client='topcat)
]]></verbatim>
have exactly the same meaning.  It's a matter of taste which you prefer.
</p>

</subsect>

<subsect id="jytask">
<subhead><title>Tasks</title></subhead>

<p>The STILTS tasks documented in <ref id="cmdUsage"/>
can be used under their usual names if they are imported from the
<code>stilts</code> module.
STILTS parameters as are supplied as named arguments of the python
functions.  In general they are either table objects for table input
parameters or strings, but in some cases python arrays are accepted,
and numbers may be used where appropriate.
The STILTS input format (<code>ifmt</code>, <code>istream</code>),
filter (<code>cmd</code>/<code>icmd</code>/<code>ocmd</code>)
and output mode (<code>omode</code>) parameters are not used however;
instead perform filtering directly on the table inputs and outputs
using the python <code>cmd_*</code> and <code>mode_*</code>
table methods or functions.
</p>

<p>Here is an example of concatenating two similar tables together
and writing the result:
<verbatim><![CDATA[
   >>> from stilts import tread, tcat
   >>> t1 = tread('data1.csv', fmt='csv')
   >>> t2 = tread('data2.csv', fmt='csv')
   >>> t12 = tcat([t1,t2], seqcol='seq')
   >>> t12.write('t12.csv', fmt='csv')
]]></verbatim>
</p>

<p>Note that for those tasks which have a parameter named "<code>in</code>"
in command-line STILTS, it has been renamed as "<code>in_</code>" for
the python version, to avoid a name clash with the python reserved word.
In most cases, the <code>in</code> parameter is the first, mandatory
parameter in any case, and so can be referenced by position as in the
previous example (we could have written "<code>tcat(in_=[t1,t2])</code>"
instead).
</p>

</subsect>

<subsect id="jyfuncs">
<subhead><title>Calculation Functions</title></subhead>

<p>The various functions from the expression language 
listed in <ref id="staticMethods"/> are available directly from JyStilts.
Each of the subsections in that section is a class in the <code>stilts</code>
module namespace, with unbound functions representing the functions.
</p>

<p>This means you can use them like this:
<verbatim><![CDATA[
    >>> import stilts
    >>> print stilts.Times.mjdToIso(54292)
    2007-07-11T00:00:00
]]></verbatim>
or like this:
<verbatim><![CDATA[
    >>> from stilts import CoordsDegrees
    >>> dist = CoordsDegrees.skyDistanceDegrees(ra1, dec1, ra2, dec2)
]]></verbatim>
</p>

</subsect>

</sect>


<sect id="io">
<subhead><title>Table I/O</title></subhead>

<p>Most of the tools in this package either read one or more tables as input, 
or write one or more tables as output, or both.
This section explains what kind of tables the tools can read and write,
and how you tell them where to find the tables to operate on.
</p>

<p>In most cases input and output table specifications are given
by parameters with the following names (or similar ones):
<dl>
<dt><code>in</code></dt>
<dd><p>Location of the input table;
    usually a filename or URL,
    but maybe some other source of data (see <ref id="location-syntax"/>)
    or a <em>scheme specification</em> (see <ref id="TableScheme"/>)
    </p></dd>
<dt><code>ifmt</code></dt>
<dd><p>Format of the input table, as listed in <ref id="inFormats"/>,
    or blank for automatic format detection based on input content or name
    (but ignored for <em>scheme specifications</em>)
    </p></dd>
<dt><code>out</code></dt>
<dd><p>Destination of the output table;
    usually a filename or "<code>-</code>"/blank for standard output
    </p></dd>
<dt><code>ofmt</code></dt>
<dd><p>Format of the output table as listed in <ref id="outFormats"/>,
    or blank for automatic format detection based on output filename
    </p></dd>
</dl>
</p>

<subsect id="ioFormats">
<subhead><title>Table Formats</title></subhead>

<p>The generic table commands in STILTS 
(currently <ref id="tpipe"><code>tpipe</code></ref>,
           <ref id="tcopy"><code>tcopy</code></ref>,
           <ref id="tmulti"><code>tmulti</code></ref>,
           <ref id="tmultin"><code>tmultin</code></ref>,
           <ref id="tcat"><code>tcat</code></ref>,
           <ref id="tcatn"><code>tcatn</code></ref>,
           <ref id="tloop"><code>tloop</code></ref>,
           <ref id="tjoin"><code>tjoin</code></ref>,
           <ref id="tgridmap"><code>tgridmap</code></ref>,
           <ref id="tgroup"><code>tgroup</code></ref>,
           <ref id="tcube"><code>tcube</code></ref>,
           <ref id="tmatch1"><code>tmatch1</code></ref>,
           <ref id="tmatch2"><code>tmatch2</code></ref>,
           <ref id="tmatchn"><code>tmatchn</code></ref>,
           <ref id="tskymap"><code>tskymap</code></ref>,
           <ref id="tskymatch2"><code>tskymatch2</code></ref>,
           <ref id="pixfoot"><code>pixfoot</code></ref>,
           <ref id="pixsample"><code>pixsample</code></ref>,
           <ref id="plot2corner"><code>plot2corner</code></ref>,
           <ref id="plot2cube"><code>plot2cube</code></ref>,
           <ref id="plot2plane"><code>plot2plane</code></ref>,
           <ref id="plot2sky"><code>plot2sky</code></ref>,
           <ref id="plot2sphere"><code>plot2sphere</code></ref>,
           <ref id="plot2time"><code>plot2time</code></ref>,
           <ref id="plot2d"><code>plot2d</code></ref>,
           <ref id="plot3d"><code>plot3d</code></ref>,
           <ref id="plothist"><code>plothist</code></ref>,
           <ref id="cdsskymatch"><code>cdsskymatch</code></ref>,
           <ref id="cone"><code>cone</code></ref>,
           <ref id="coneskymatch"><code>coneskymatch</code></ref>,
           <ref id="sqlskymatch"><code>sqlskymatch</code></ref>,
           <ref id="tapquery"><code>tapquery</code></ref>,
           <ref id="tapresume"><code>tapresume</code></ref>,
           <ref id="tapskymatch"><code>tapskymatch</code></ref> and
           <ref id="regquery"><code>regquery</code></ref>)
have no native format for table storage, they can process
data in a number of formats equally well.  
STIL has its own model of what a table
consists of, which is basically:
<ul>
<li>Some per-table metadata (parameters)</li>
<li>A number of columns</li>
<li>Some per-column metadata</li>
<li>A number of rows, each containing one entry per column</li>
</ul>
Some table formats have better facilities for storing this sort of
thing than others, and when performing conversions STILTS does 
its best to translate between them, but it can't perform the
impossible: for instance there is nowhere in a Comma-Separated Values 
file to store descriptions of column units, 
so these will be lost when converting from VOTable to CSV formats.
</p>

<p>The formats the package knows about are dependent on the input and
output handlers currently installed.  The ones installed by default
are listed in the following subsections.  More may be added in the 
future, and it is possible to install new ones at runtime - see 
the <docxref doc="sun252">STIL documentation</docxref> for details.
</p>

<p>Some formats can be used to hold multiple tables in a single file,
and others can only hold a single table per file.
</p>

<subsubsect id="inFormats">
<subhead><title>Input Formats</title></subhead>

<p>Some of the tools in this package ask you to specify the format
of input tables using the <code>ifmt</code> (or a similarly named)
parameter.
For some file formats (e.g. FITS, VOTable, CDF),
the format can be automatically determined by
looking at the file content, regardless of filename;
for others (e.g. CSV files with a "<code>.csv</code>" extension),
STILTS may be able to use the filename as a hint to guess the format
(the details of these rules are given in the format-specific
subsections below).
Otherwise, you have to supply the format using the <code>ifmt</code> parameter.
It is always safe to specify the format explicitly;
this will be slightly more efficient than auto-determination,
and may lead to more helpful error messages in the case that the
table can't be read correctly.
</p>

<p>The available input formats are described in the following subsections.
It is also possible to add new formats at runtime using the
<code>startable.readers</code> <ref id="sysProperties">system property</ref>,
or by setting the format to the classname of a
<code>uk.ac.starlink.table.TableBuilder</code> class.
</p>

<subsubsubsect id="inFits">
<subhead><title><code>fits</code></title></subhead>
&in-fits;

<p>There are actually two FITS input handlers,
<code>fits-basic</code> and <code>fits-plus</code>.
The <code>fits-basic</code> handler extracts standard column metadata
from FITS headers of the HDU in which the table is found,
while the <code>fits-plus</code> handler reads column and table metadata
from VOTable content stored in the primary HDU of the multi-extension
FITS file.
FITS-plus is a private convention effectively defined by the
corresponding output handler; it allows de/serialization of
much richer metadata than can be stored in standard FITS headers
when the FITS file is read by fits-plus-aware readers,
though other readers can understand the unenhanced FITS file perfectly well.
It is normally not necessary to worry about this distinction;
STILTS will determine whether a FITS file is FITS-plus or not based on its
content and use the appropriate handler, but if you want to force the
reader to use or ignore the enriched header, you can explicitly specify
an input format of "<code>fits-plus</code>" or "<code>fits-basic</code>".
The details of the FITS-plus convention are described in <ref id="fitsPlus"/>.
</p>
</subsubsubsect>

<subsubsubsect id="inColfits">
<subhead><title><code>colfits</code></title></subhead>
&in-colfits;

<p>Like the normal (row-oriented) <ref id="inFits">FITS handler</ref>,
two variants are supported:
with (<code>colfits-plus</code>) or without (<code>colfits-basic</code>)
metadata stored as a VOTable byte array in the primary HDU.
For details of the FITS-plus convention, see <ref id="fitsPlus"/>.
</p>
</subsubsubsect>

<subsubsubsect id="inVotable">
<subhead><title><code>votable</code></title></subhead>
&in-votable;
</subsubsubsect>

<subsubsubsect id="inCdf">
<subhead><title><code>cdf</code></title></subhead>
&in-cdf;
</subsubsubsect>

<subsubsubsect id="inCsv">
<subhead><title><code>csv</code></title></subhead>
&in-csv;
<p>See also <ref id="inEcsv">ECSV</ref> as a format which is
similar and capable of storing more metadata.
</p>
</subsubsubsect>

<subsubsubsect id="inEcsv">
<subhead><title><code>ecsv</code></title></subhead>
&in-ecsv;
</subsubsubsect>

<subsubsubsect id="inAscii">
<subhead><title><code>ascii</code></title></subhead>
&in-ascii;
</subsubsubsect>

<subsubsubsect id="inIpac">
<subhead><title><code>ipac</code></title></subhead>
&in-ipac;
</subsubsubsect>

<subsubsubsect id="inPds4">
<subhead><title><code>pds4</code></title></subhead>
&in-pds4;
</subsubsubsect>

<subsubsubsect id="inMrt">
<subhead><title><code>mrt</code></title></subhead>
&in-mrt;
</subsubsubsect>

<subsubsubsect id="inParquet">
<subhead><title><code>parquet</code></title></subhead>
&in-parquet;
</subsubsubsect>

<subsubsubsect id="inHapi">
<subhead><title><code>hapi</code></title></subhead>
&in-hapi;
</subsubsubsect>

<subsubsubsect id="inFeather">
<subhead><title><code>feather</code></title></subhead>
&in-feather;
</subsubsubsect>

<subsubsubsect id="inGbin">
<subhead><title><code>gbin</code></title></subhead>
&in-gbin;

<p><strong>Example:</strong>
Suppose you have the <code>MDBExplorerStandalone.jar</code> file
containing the data model classes, you can read GBIN files by
starting STILTS like this:
<verbatim>
   stilts -classpath MDBExplorerStandalone.jar ...
</verbatim>
or like this:
<verbatim>
   java -classpath stilts.jar:MDBExplorerStandalone.jar uk.ac.starlink.ttools.Stilts ...
</verbatim>
</p>
</subsubsubsect>

<subsubsubsect id="inTst">
<subhead><title><code>tst</code></title></subhead>
&in-tst;
</subsubsubsect>

<subsubsubsect id="inWdc">
<subhead><title><code>wdc</code></title></subhead>
&in-wdc;
</subsubsubsect>

</subsubsect>

<subsubsect id="outFormats">
<subhead><title>Output Formats</title></subhead>

<p>Some of the tools ask you to specify the format of output tables
using the <code>ofmt</code> parameter.
The output formats described below are supported; in some cases there
are variants or options for the basic formats as documented.
If you don't specify an output format explicitly,
STILTS will try to guess what format to write based on the
output filename; the details of those rules are also documented below.
</p>

<p>It is also possible to add new formats at runtime using the
<code>startable.writers</code> <ref id="sysProperties">system property</ref>,
or by setting the format to the classname of a
<code>uk.ac.starlink.table.StarTableWriter</code> class.
</p>

<subsubsubsect id="outFits">
<subhead><title><code>fits</code></title></subhead>
&out-fits;
</subsubsubsect>

<subsubsubsect id="outVotable">
<subhead><title><code>votable</code></title></subhead>
&out-votable;
</subsubsubsect>

<subsubsubsect id="outCsv">
<subhead><title><code>csv</code></title></subhead>
&out-csv;
</subsubsubsect>

<subsubsubsect id="outEcsv">
<subhead><title><code>ecsv</code></title></subhead>
&out-ecsv;
</subsubsubsect>

<subsubsubsect id="outAscii">
<subhead><title><code>ascii</code></title></subhead>
&out-ascii;
</subsubsubsect>

<subsubsubsect id="outIpac">
<subhead><title><code>ipac</code></title></subhead>
&out-ipac;
</subsubsubsect>

<subsubsubsect id="outParquet">
<subhead><title><code>parquet</code></title></subhead>
&out-parquet;
</subsubsubsect>

<subsubsubsect id="outFeather">
<subhead><title><code>feather</code></title></subhead>
&out-feather;
</subsubsubsect>

<subsubsubsect id="outText">
<subhead><title><code>text</code></title></subhead>
&out-text;
</subsubsubsect>

<subsubsubsect id="outHtml">
<subhead><title><code>html</code></title></subhead>
&out-html;
</subsubsubsect>

<subsubsubsect id="outLatex">
<subhead><title><code>latex</code></title></subhead>
&out-latex;
</subsubsubsect>

<subsubsubsect id="outTst">
<subhead><title><code>tst</code></title></subhead>
&out-tst;
</subsubsubsect>

<subsubsubsect id="outMirage">
<subhead><title><code>mirage</code></title></subhead>
&out-mirage;
</subsubsubsect>

</subsubsect>

<subsubsect id="fitsConventions">
<subhead><title>Non-standard FITS conventions</title></subhead>

<p>STIL, the I/O library underlying STILTS,
uses a few private conventions when writing and reading FITS files.
These are not private in the sense that non-STIL code is prevented from
cooperating with them, but STIL does not assume that other code,
or FITS tables it encounters, will use these conventions.
Instead, they offer (in some cases) added value for tables that were
written by STIL and are subsequently re-read by STIL, while causing
the minimum of trouble for non-STIL readers.
</p>

<subsubsubsect id="fitsPlus">
<subhead><title>FITS-plus</title></subhead>
&fits-plus;
</subsubsubsect>

<subsubsubsect id="wideFits">
<subhead><title>Wide FITS</title></subhead>
&fits-wide;
</subsubsubsect>

</subsubsect>

</subsect>

<subsect id="location-syntax">
<subhead><title>Input Locations</title></subhead>

<p>The location of a serialized input table,
usually given using the <code>in</code> parameter or similar,
may be given in one of the forms listed below.
<dl>

<dt>Filename</dt>
<dd><p>Very often, you will simply specify a filename as location, and
    the tool will just read from it in the usual way.
    </p></dd>

<dt>URL</dt>
<dd><p>Tables can be read from URLs directly.
    Some non-standard URL protocols are supported
    as well as the usual ones.  The list is:
    <dl>
    <dt><code>http:</code></dt>
    <dd><p>Read from HTTP resources.</p></dd>
    <dt><code>https:</code></dt>
    <dd><p>Read from HTTPS resources.</p></dd>
    <dt><code>ftp:</code></dt>
    <dd><p>Read from anonymous FTP resources.</p></dd>
    <dt><code>file:</code></dt>
    <dd><p>Read from local files, using the syntax
        <code>file:///path/to/file</code>.
        This is similar to specifying the filename directly, but
        there is a difference: using this form forces reads to be sequential
        rather than random access, which may allow you to experience
        a different set of performance characteristics and bugs.</p></dd>
    <dt><code>jar:</code></dt>
    <dd><p>Specialised protocol for looking inside Java Archive files -
        see <webref url="&URL.J2SE_DOCS;api/java/net/JarURLConnection.html"
                    >JarURLConnection</webref> documentation.</p></dd>
    <dt><code>myspace:</code></dt>
    <dd><p><em>(Obsolete?)</em>
        Accesses files in the AstroGrid "MySpace" virtual file store.
        These URLs look something like
        "<code>myspace:/survey/iras_psc.xml</code>",
        and can access files in the myspace are that the user is currently
        logged into.
        These URLs can be used for both input and output of tables.
        To use them you must have an AstroGrid account and the AstroGrid
        WorkBench or similar must be running; if you're not currently
        logged in a dialogue will pop up to ask you for name and
        password.</p></dd>
    <dt><code>ivo:</code></dt>
    <dd><p><em>(Obsolete?)</em>
        Understands ivo-type URLs which signify files in the
        AstroGrid "MySpace" virtual file store.
        These URLs look something like
        "<code>ivo://uk.ac.le.star/filemanager#node-2583</code>".
        These URLs can be used for both input and output of tables.
        To use them you must have an AstroGrid account and the AstroGrid
        WorkBench or similar must be running; if you're not currently
        logged in a dialogue will pop up to ask you for name and
        password.</p></dd>
    <dt><code>jdbc:</code></dt>
    <dd><p>JDBC URLs may be used, but they don't work in the same way
        as the others listed here, since they do not reference an
        input byte stream.  See instead <ref id="scheme-jdbc"/>.</p></dd>
    </dl>
    </p></dd>

<dt>Minus sign ("<code>-</code>")</dt>
<dd><p>The special location "-" (minus sign) indicates standard input.
    This allows you to use STILTS commands in a normal Unix pipeline.
    </p></dd>

<dt>System command ("<code>&lt;</code><em>syscmd</em>" or
                    "<em>syscmd</em><code>|</code>")</dt>
<dd><p>If the location starts with a "<code>&lt;</code>" character
    or ends with a "<code>|</code>" character,
    the rest of the string is taken as a command line to be executed
    by the system shell.
    For instance a location like
    "<code>&lt;cat header.txt data.txt</code>"
    (or equivalently
    "<code>cat header.txt data.txt|</code>")
    could be used to prepend a header line to an ASCII data file
    before it is passed to the STILTS ASCII-format input handler.
    Note this syntax will probably only work on Unix-like systems.
    </p></dd>
</dl>
</p>

<p>In any of these cases, for input locations compression is taken care
of automatically.  That means that you can give the filename or URL
of a file which is compressed using <code>gzip</code>, <code>bzip2</code>
or Unix <code>compress</code> and the program will uncompress it on the fly.
</p>

<p>For file formats that can contain multiple tables,
the one required, if it's not the first in the file,
can generally be specified by a position indicator string
appended to the basic location following a "<code>#</code>".
For instance "cat.fits#3" references HDU 3 in a multi-extension FITS file.
The details of this syntax depend on the file format,
and are given in the relevant subsection of
<ref id="inFormats"/>.
</p>

<p>Note that tables can also be supplied from non-serialized sources, 
as described in <ref id="TableScheme">Input Schemes</ref>.
</p>

</subsect>


<subsect id="TableScheme">
<subhead><title>Input Schemes</title></subhead>

<p>As well as being able to load tables from external data streams,
STILTS offers a way to specify tables that do not correspond to a
stream of bytes.  These may be defined programmatically or interact
with external services in some way that is not as straightforward
as decoding a stream of bytes.
</p>

<p>Such tables are defined using different <em>schemes</em>,
and scheme specifications may be used in the same places as
input table names, for instance as the value of the <code>in</code>
parameter in <ref id="tpipe"><code>tpipe</code></ref> and other commands.
If an input location parameter (<code>in</code>) uses a scheme specification,
the corresponding input format parameter (<code>ifmt</code>) is ignored.
</p>

<p>The form of a scheme specification is:
<verbatim>
   :&lt;scheme-name&gt;:&lt;scheme-specific-part&gt;
</verbatim>
so that for instance "<code>in=:loop:10</code>" specifies
a 10-row single-column table,
as described by the <ref id="scheme-loop">loop</ref> scheme documentation
below.
For an example of using such tables, you can try running for instance
<verbatim>
   stilts plot2plane in=:attractor:1e7,clifford layer1=mark shading1=density densemap1=plasma x1=x y1=y
</verbatim>
</p>

<p>The following subsections describe all the schemes that are
available by default.
It is also possible to add new schemes at runtime by using the
<code>startable.schemes</code> <ref id="sysProperties">system property</ref>. 
</p>

&scheme-docs;

</subsect>

<subsect id="AuthManager">
<subhead><title>Authentication</title></subhead>

<p>Some external data services restrict access to registered users,
meaning that you have to log in to use them.
If STILTS encounters such a restriction and knows how to try to
authenticate to the service in question,
it will report on the console the URL for which
the access is blocked (and possibly some additional information
about the way authentication is being carried out),
and ask for entry of a username and password.
If authentication is successful, the resource can be retrieved,
and so can any other resources from the same place,
so if multiple contacts to the same service
are required from the same STILTS command/session,
only one login attempt should be required.
</p>

<p>If user interaction during the command is not suitable,
it is possible to supply a username and password using the
<ref id="sysProperties">system properties</ref>
<code>auth.username</code> and <code>auth.password</code>.
If both of these are set, then instead of asking on the console
for login credentials, they will be taken from the property values,
for instance
<verbatim>
   stilts -Dauth.username=foo -Dauth.password=@~/passwd.txt
          tpipe in=https://secret.com/data.vot
</verbatim>
would access the named resource, and if challenged by the service
for authentication would supply "<code>foo</code>" for user name
and the contents of the file at <code>~/passwd.txt</code> as the password.
Note however that this feature should be used with care,
since it passes private information indiscriminately to any
service that asks for it.
</p>

<p>Some TAP services offer optional authentication;
anonymous access is permitted, but users who log in may
benefit from restricted data or enhanced resource limits.
By default, the TAP access commands
(<ref id="tapquery"><code>tapquery</code></ref>,
 <ref id="taplint"><code>taplint</code></ref>,
 <ref id="tapskymatch"><code>tapskymatch</code></ref> and
 <ref id="tapresume"><code>tapresume</code></ref>)
will use anonymous access by default in this case.
But if you prefer to use the service in authenticated mode,
you can supply the <code><ref id="tapquery-auth">auth</ref>=true</code>
parameter and an attempt will be made to log in before use.
</p>

<p>There is currently no way to log in to non-TAP VO services
that provide optional authentication;
however at time of writing I'm not aware of any.
</p>

<p><strong>Note:</strong>
These authentication arrangements in STILTS are new at version 3.4-9,
and rely on VO standards that are still under discussion.
The behaviour and user interface may change in future releases,
and at time of writing not all data services that provide authentication
advertise it in a way that STILTS can work with.
It is hoped that authentication interoperability
will improve in future versions of STILTS and of server-side software.
</p>

</subsect>

</sect>


<sect id="pipes">
<subhead><title>Table Pipelines</title></subhead>

<p>Several of the tasks available in STILTS take one or more input tables,
do something or other with them, and produce one or more output tables.
This is a pretty obvious way to go about things, and in the most
straightforward case that's exactly what happens: you name one
or more input tables,
specify the processing parameters, and name an output table;
the task then reads the input tables from disk, does the processing
and writes the output table to disk.
</p>

<p>However, many of the tasks in STILTS allow you to do pre-processing
of the input tables before the main job, post-processing of the
output table after the main job, and to decide what happens to 
the final tabular result, without any intermediate storage of the data.  
Examples of the kind of pre-processing
you might want to do are to rearrange the columns so that they
have the right units for the main task, or replace 'magic' values
such as -999 with genuine blank values; the kind of post-processing
you might want to do is to sort the rows in the output table or
delete some of the columns you're not interested in.
As for the destination of the final table, you might want to 
write it to disk, but equally you might not want to store it anywhere,
but only be interested in counting the number of rows, or seeing
the minima/maxima of a few of the columns, or you might want to
send it straight to TOPCAT or some other table viewing application
for interactive analysis.
</p>

<p>Clearly, you could achieve the same effect by running multiple
applications: preprocess your 
original input tables to write intermediate files on disk, 
run the main processing application which reads those files
from disk and writes a new output file,
run another application to postprocess the output file and
write a new final output file,
and finally do something with this such as counting the rows in it
or viewing it in TOPCAT.
However, by doing it all within a single task instead,
no intermediate results have to be stored,
and the whole sequence can be very much more efficient.
You can think of this (if it helps) like a Unix pipeline, 
except what is being streamed from the start to the end of the
pipe is not bytes, but table metadata and data.
In most cases, the table data is streamed through the pipeline a
row at a time, meaning that the amount of memory required is small
(though in some cases, for instance row sorting and crossmatching, 
this is not possible).
</p>

<p>Tasks which allow this pre/post-processing, or "filtering",
have parameters with names like "<code>cmd</code>" which you
use to specify processing steps.
Tasks with multiple input tables
(<ref id="tmatch2"><code>tmatch2</code></ref>,
 <ref id="tskymatch2"><code>tskymatch2</code></ref>,
 <ref id="tcatn"><code>tcatn</code></ref>,
 <ref id="tjoin"><code>tjoin</code></ref>)
may have parameters named <code>icmd1</code>, <code>icmd2</code>, ...
for preprocessing the different input tables and
<code>ocmd</code> for postprocessing the output table.
<ref id="tpipe"><code>tpipe</code></ref> does nothing except
filtering, so there is no distinction between pre- and post-processing,
and its filter parameter is just named <code>cmd</code>.
<code>tpipe</code> additionally has a <code>script</code>
parameter which allows you to use a text file to write the
commands in, to prevent the command line getting too long.
In both cases there is a parameter named <code>omode</code>
which defines the "output mode", that is, what happens to the
post-processed output table that comes out of the end of the pipeline.
</p>

<p><ref id="filterSteps"/> lists the processing steps available,
and explains how to use them,
<ref id="col-id"/> and <ref id="colid-list"/> describe the syntax
used in some of these filter commands for specifying columns,
and <ref id="outModes"/> describes the available output modes.
See the examples in the
<ref id="cmdUsage">command reference</ref>,
and particularly the
<ref id="tpipeExamples" plaintextref="yes"><code>tpipe</code> examples</ref>,
for some examples putting all this together.
</p>

<subsect id="filterSteps">
<subhead><title>Processing Filters</title></subhead>

<p>This section lists the filter commands which can be used for 
table pipeline processing, in conjunction with <code>cmd</code>-
or <code>script</code>-type parameters.
</p>

<p>You can string as many of these together as you like.
On the command line, you can repeat the <code>cmd</code>
(or <code>icmd1</code>, or <code>ocmd</code>...) parameter 
multiple times, or use one <code>cmd</code> parameter and
separate different filter specifiers with semicolons ("<code>;</code>").
The effect is the same.
</p>

<p>It's important to note that each command in the sequence of
processing steps acts on the table at that point in the sequence.
Thus either of the two identical invocations:
<verbatim>
   stilts tpipe cmd='delcols 1; delcols 1; delcols 1'
   stilts tpipe cmd='delcols 1' cmd='delcols 1' cmd='delcols 1'
</verbatim>
has the same effect as
<verbatim>
   stilts tpipe cmd='delcols "1 2 3"'
</verbatim>
since in the first case the columns are shifted left after
each one is deleted, so the table seen by each step has one fewer
column than the one before.
Note also the use of quotes in the latter of the examples above,
which is necessary so that the <code>&lt;colid-list&gt;</code>
of the <code>delcols</code> command is interpreted as one argument not
three separate words.
</p>

<p>The available filters are described in the following subsections.
</p>

&filter-docs;

</subsect>

<subsect id="col-id">
<subhead><title>Specifying a Single Column</title></subhead>

<p>If an argument is specified in the help text for a 
command with the symbol <code>&lt;col-id&gt;</code>
it means you must give a string which identifies one of the
existing columns in a table.
</p>

<p>There are several ways you can specify a column in this context:
<dl>
<dt>Column Name</dt>
<dd><p>The name of the column may be used if it contains no spaces.
    It is usually matched case insensitively.  If multiple columns
    have the same name, the first one that matches is selected.
    </p></dd>

<dt>Column Index or &column.id.char;ID</dt>
<dd><p>The index of the column may always be used; this is a useful
    fallback if the column name isn't suitable for some reason.
    The first column is '1', the second is '2' and so on.
    You may alternatively use the forms 
    '&column.id.char;1', '&column.id.char;2' etc.
    </p>
    <p>Tip: if counting which column has which index is giving you a
    headache, running <code>tpipe</code> with <code>omode=meta</code> or 
    <code>omode=stats</code> on the table may help.
    </p></dd>

<dt>Column &ucd.prefix; specifier</dt>
<dd><p>If the column has a
    <webref url="http://www.ivoa.net/Documents/latest/UCD.html"
            >Unified Content Descriptor</webref>
    (this will usually only be the case for VOTable or possibly FITS format
    tables) you can refer to it using an identifier of the form
    "<code>&ucd.prefix;&lt;ucd-spec&gt;</code>".  Depending on the version of
    UCD scheme used, UCDs can contain various punctuation marks such
    as underscores, semicolons and dots; for the purpose of this syntax
    these should all be represented as underscores ("<code>_</code>").
    So to identify a column which has the UCD "<code>phot.mag;em.opt.R</code>",
    you should use the identifier "<code>&ucd.prefix;phot_mag_em_opt_r</code>".
    Matching is not case-sensitive.  Futhermore, a trailing underscore
    acts as a wildcard, so that the above column could also be referenced
    using the identifier "<code>&ucd.prefix;phot_mag_</code>".  If multiple 
    columns have UCDs which match the given identifer, the first one
    will be used.
    </p></dd>

<dt>Column &utype.prefix; specifier</dt>
<dd><p>If the column has a <label>Utype</label>
    (this will usually only be the case for VOTable or possibly FITS format
    tables) you can refer to it using an identifier of the form
    "<code>&utype.prefix;&lt;utype-spec&gt;</code>".
    Utypes may contain various punctuation marks such as colons and dots;
    for the purpose of this syntax these should all be represented as
    underscores ("<code>_</code>").
    So to identify a column which has the Utype 
    "<code>ssa:Access.Format</code>",
    you should use the identifier 
    "<code>&utype.prefix;ssa_Access_format</code>".
    Matching is not case-sensitive.
    If multiple columns have Utypes which match the given identifier,
    the first one will be used.
    </p></dd>
</dl>
</p>

</subsect>

<subsect id="colid-list">
<subhead><title>Specifying a List of Columns</title></subhead>

<p>If an argument is specified in the help text for a command
with the symbol <code>&lt;colid-list&gt;</code> it means you 
must give a string which identifies a list of zero, one or more
of the existing columns in a table.
The string you specify is a separated into separate tokens by
whitespace, which means that you will normally 
have to surround it in single or double quotes to ensure
that it is treated as a single argument and not several of them.
</p>

<p>Each token in the <code>&lt;colid-list&gt;</code> string may be one of
the following:
<dl>
<dt>Single Column Identifier</dt>
<dd><p>The identifier for a single column, as described in <ref id="col-id"/>:
    one of
    <code>&lt;name&gt;</code>,
    <code>&lt;index&gt;</code>,
    <code>&column.id.char;&lt;index&gt;</code>,
    <code>&ucd.prefix;&lt;ucd-expr&gt;</code> or
    <code>&utype.prefix;&lt;utype-expr&gt;</code>.
    </p></dd>

<dt>Wildcard Expression</dt>
<dd><p>You can use a simple form of wildcard expression which expands
    to any columns in the table whose names match the pattern.
    Currently, the only special character is an asterisk '<code>*</code>'
    which matches any sequence of characters.  To match an unknown
    sequence at the start or end of the string an asterisk must be
    given explicitly.  Other than that, matching is usually 
    case insensitive.  The order of the expanded list is the same
    as the order in which the columns appear in the table.
    </p>

    <p>Thus "<code>col*</code>" will match columns named 
    <code>col1</code>, <code>Column2</code> and <code>COL_1024</code>,
    but not <code>decOld</code>.
    "<code>*MAG*</code>" will match columns named
    <code>magnitude</code>, <code>ABS_MAG_U</code> and <code>JMAG</code>.
    "<code>*</code>" on its own 
    expands to a list of all the columns of the table in order.
    </p></dd>

<dt>Column Range</dt>
<dd><p>You can specify a range of columns in order using a token of the form
    <code>&lt;first-colid&gt;-&lt;last-colid&gt;</code>,
    where the syntax for
    <code>&lt;first-colid&gt;</code> and <code>&lt;last-colid&gt;</code>
    is a <label>Single Column Identifier</label> as above
    (as long as it doesn't contain a "<code>-</code>" character).
    The range is inclusive, so the first and last column are both included.
    The <code>&lt;first-colid&gt;</code> or <code>&lt;last-colid&gt;</code>
    part (but not both) may be omitted, to indicate all the columns from the
    start or all the columns to the end, respectively.
    </p>

    <p>So "<code>RA-PARALLAX</code>" means all the columns starting with
    the one named <code>RA</code> and ending with
    the one named <code>PARALLAX</code> (inclusive);
    "<code>1-100</code>" or "<code>&column.id.char;1-&column.id.char;100</code>"
    means the first hundred columns in the table;
    "<code>101-</code>" or "<code>&column.id.char;101-</code>"
    means all the columns apart from the first hundred,
    "<code>PARALLAX-</code>" means <code>PARALLAX</code> and all
    subsequent columns, etc.
    </p></dd>

</dl>
</p>

<p>Specifying a list which contains a given column more than once is
not usually an error, but what effect it has depends on the 
function you are executing.
</p>

</subsect>

<subsect id="outModes">
<subhead><title>Output Modes</title></subhead>

<p>This section lists the output modes which can be used as
the value of the <code>omode</code> parameter of 
<code><ref id="tpipe">tpipe</ref></code> and other commands.
Typically, having produced a result table by pipeline processing 
an input one, you will write it out by specifying 
<code>omode=out</code> (or not using the <code>omode</code> parameter at all -
<code>out</code> is the default).  However, you can do other things
such as calculate statistics, display metadata, etc.  In some of
these cases, additional parameters are required.  The different 
output modes, with their associated parameters, are described in
the following subsections.
</p>

&mode-docs;

</subsect>


</sect>


<sect id="match">
<subhead><title>Crossmatching</title></subhead>

<p>STILTS offers flexible and efficient facilities for crossmatching tables.
Crossmatching is identifying different rows, which may be in the
same or different tables, that refer to the same item.
In an astronomical context such an item is usually, 
though not necessarily, an astronomical source or object.
This operation corresponds to what in database terminology 
is called a <em>join</em>.
</p>

<p>There are various complexities to specifying such a match.
In the first place you have to define what is the condition that
must be satisfied for two rows to be considered matching.
In the second place you must decide what happens if, for a given row,
more than one match can be found.  Finally, you have to decide what
to do having worked out what the matched rows are; the result will
generally be presented as a new output table, but there are various
choices about what columns and rows it will consist of.
Some of these issues are discussed in this section, and others
in the reference sections on the tools themselves in <ref id="cmdUsage"/>.
</p>

<p>Matching can in general be a computationally intensive process.
The algorithm used by the <code>tmatch*</code> tasks in STILTS, 
except in pathological cases, scales as <m>O(N log(N))</m> or thereabouts,
where <m>N</m> is the total number of rows in all the tables being matched.
No preparation (such as sorting) is required on the tables prior to
invoking the matching operation.
It is reasonably fast; for instance an RA, Dec positional match
of two 10<sup>5</sup>-row catalogues takes of the order of 60 seconds
on current (2005 laptop) hardware.  Attempting matches with large tables can 
lead to running out of memory; the calculation just mentioned required a
java heap size of around 200Mb (<code>-Xmx200M</code>).
<!--
   stilts -Xmx200M tmatch2 \
      in1=2dfgrs_ngp.fits \
      in2=2dfgrs_spectro.fits \
      icmd1='addskycoords -inunit rad -outunit deg fk5 fk5 \
                          RA2000 DEC2000 RA DEC' \
      icmd2='addskycoords -inunit rad -outunit deg fk5 fk5 \
                          RA2000 DEC2000 RA DEC' \
      matcher=sky params=3 values1='RA DEC' values2='RA DEC' \
      join=1and2 find=best \
      ocmd='addcol RA_DIFF ra_1-ra_2; addcol DEC_DIFF dec_1-dec_2' \
      omode=stats
 -->
</p>

<p>In the current release of STILTS the following tasks are provided
for crossmatching between local tables:
<dl>
<dt><ref id="tmatch2"><code>tmatch2</code></ref></dt>
<dd><p>Generic crossmatching between two tables.
    </p></dd>

<dt><ref id="tskymatch2"><code>tskymatch2</code></ref></dt>
<dd><p>Crossmatching between two tables where the matching criterion is
    a fixed separation on the sky.  This is simply a stripped-down 
    version of <code>tmatch2</code> provided for convenience when the
    full generality is not required.
    </p></dd>

<dt><ref id="tmatch1"><code>tmatch1</code></ref></dt>
<dd><p>Generic crossmatching internal to a single table.
    The basic task this performs is to identify groups of rows within a
    single table which match each other. 
    </p></dd>

<dt><ref id="tmatchn"><code>tmatchn</code></ref></dt>
<dd><p>Generic crossmatching between multiple (&gt;2) tables.
    </p></dd>

<dt><ref id="tjoin"><code>tjoin</code></ref></dt>
<dd><p>Trivial join operation between multiple tables in which no row
    re-ordering is required.  This barely warrants the term "crossmatch"
    and the concepts explained in the rest of this section are not
    relevant to it.
    </p></dd>
</dl>
</p>

<subsect id="MatchEngine">
<subhead><title>Match Criteria</title></subhead>

<p>Determining whether one row represents the same item as another
is done by comparing the values in certain of their columns to see
if they are the same or similar.
The most common astronomical case is to say that two rows match if
their celestial coordinates (right ascension and declination) are
within a given small radius of each other on the sky.
There are other possibilities; for instance the coordinates to compare
may be in a Cartesian space, or have a higher (or lower) dimensionality
than two, or the match may be exact rather than within an error radius....
</p>

<p>If you just need to match two tables according to sky position
with fixed errors you are recommended to use the simplified
<ref id="tskymatch2"><code>tskymatch2</code></ref> task.
For other cases, this section describes how to specify much more
flexible match criteria for use with
<code>tmatch1</code>, <code>tmatch2</code> or <code>tmatchn</code>
by setting the following parameters:
<dl>
<dt><code>matcher</code></dt>
<dd><p>Name of the match criteria type.
    </p></dd>
<dt><code>params</code></dt>
    <dd><p>Fixed value(s) giving the parameters of the match 
    (typically an error radius).  If more than one value is required,
    the values should be separated by spaces.
    </p></dd>
<dt><code>values*</code></dt>
    <dd><p>Expressions to be compared between rows.  This will typically
    contain the names of one or more columns, but each element may be
    an algebraic expression (see <ref id="jel"/>) rather than just a 
    column name if required.
    If more than one value is required, the values should be separated
    by spaces.
    There is one of these parameters for each table taking part in the 
    match, so for <code>tmatch2</code> you must specify both 
    <code>values1</code> and <code>values2</code>.
    </p></dd>
<dt><code>tuning</code></dt>
    <dd><p>Fixed value(s) supplying tuning parameters for the match
    algorithm.  If there is more than one value, they should be
    separated by spaces.
    This value will have a sensible default, so you do not
    need to supply it, but providing adjusted values may make your match
    run faster or require less memory (or the reverse).
    Adjusting tuning parameters will not change the result of any match,
    only the resources required to run it.
    Looking at the progress output of a match will indicate what 
    tuning values have been used; adjusting the value a bit up or down
    is a good way to experiment.
    </p></dd>
</dl>
</p>

<p>For example, suppose we wish to locate objects in two tables which are
within 3 arcseconds of each other on the sky.  One table has columns
RA and DEC which give coordinates in degrees, and the other has columns
RArad and DECrad which give coordinates in radians.  These are the
arguments which would be used to tell <code>tmatch2</code> what the match
criteria are:
<verbatim>
   matcher=sky
   params=3
   values1='RA DEC'
   values2='radiansToDegrees(RArad) radiansToDegrees(DECrad)'
</verbatim>
It is clearly important that corresponding values are comparable
(in the same units) between the tables being matched,
and in geometrically sensitive cases such as matching
on the sky, it's important that they are the units expected by the
matcher as well.  To determine what those units are, either consult
the roster below, or run the following command:
<verbatim>
   stilts tmatch2 help=matcher
</verbatim>
which will tell you about all the known matchers and their associated
<code>params</code>, <code>values*</code> and <code>tuning</code> parameters.
</p>

<p>The following subsections list the basic <code>matcher</code> types and the
requirements of their associated <code>params</code>,
<code>values*</code> and <code>tuning</code> parameters.
The units of the required values are given where significant.
</p>

<subsubsect id="SkyMatchEngine">
<subhead><title><code>sky</code>: Sky Matching</title></subhead>

<p>&matcher-sky-usage;</p>

<p>The <code>sky</code> matcher compares positions on the celestial sphere
with a fixed error radius.
Rows are considered to match when the two (<code>ra</code>, <code>dec</code>)
positions are within <code>max-error</code> arcseconds of each other
along a great circle.
</p>

<p>In fact this matching is not restricted to equatorial coordinates - 
the <code>ra</code> and <code>dec</code> parameters may represent
any longitude-like and latitude-like coordinates in degrees, 
since the spherical geometry for the matching is unchanged under 
such transformations.
</p>

</subsubsect>

<subsubsect id="SkyMatchEngine-err">
<subhead><title><code>skyerr</code>:
         Sky Matching with Per-Object Errors</title></subhead>

<p>&matcher-skyerr-usage;</p>

<p>The <code>skyerr</code> matcher compares positions on the celestial 
sphere using error radii which can be different for each row.
Rows are considered to match when the separation between the
two <code>ra</code>, <code>dec</code> positions is no larger than 
the sum of the two per-row <code>error</code> values.
</p>

<p>The <code>scale</code> parameter should be a rough average value
of the error distances.  It is used only to set a sensible default for 
<code>healpix-k</code> tuning parameter, and its value does not affect
the result.  If you set <code>healpix-k</code> directly, its value is ignored.
</p>

<p>As with <ref id="SkyMatchEngine"><code>sky</code></ref> matching,
other longitude/latitude coordinate pairs may be used in place of
right ascension and declination.
</p>

<p>A variant form <code>skyerr_q</code>
does the same thing but combines the two per-row <code>error</code> values
in quadrature rather than by summation.
In that case the separation between the two positions
must be no larger than <m>sqrt(error1<sup>2</sup>+error2<sup>2</sup>)</m>.
</p>

<p><strong>Note:</strong> the semantics of this matcher have changed
slightly at version 2.4 of STILTS.
In earlier versions the single parameter was named <code>max-error</code>
and provided an additional constraint on the maximum accepted separation
between matched objects.  For most uses, the old and new behaviours are
expected to give the same results, but in cases of difference, the new
behaviour is more likely what you want.
</p>

</subsubsect>

<subsubsect id="EllipseSkyMatchEngine">
<subhead><title><code>skyellipse</code>:
                Sky Matching of Elliptical Regions</title></subhead>

<p>&matcher-skyellipse-usage;</p>

<p>The <code>skyellipse</code> matcher compares elliptical regions
on the sky for overlap.  Each row has to provide five values,
giving the centre, the major and minor radii, and the position angle
of an ellipse.
Rows are considered to match if there is any overlap between the ellipses.
The goodness of match is a normalised generalisation of the symmetrical case
used by the <code>skyerr</code> matcher, in which the best possible match
is two concentric ellipses, and the worst allowable match is when the
circumferences just touch.
</p>

<p>The calculations are approximate since in some cases they rely on
projecting the ellipses onto a Cartesian tangent plane before evaluating
the match, so for larger ellipses the criterion will be less exact.
For objects the size of most observed stars or galaxies,
this approximation is not expected to be problematic.
</p>

<p>The <code>scale</code> parameter must be supplied, and should be
a rough average value of the major radii.  it is used only to set
a sensible default for the <code>healpix-k</code> tuning parameter,
and its value does not affect the result.
If you set <code>healpix-k</code> directly,
the value of <code>scale</code> is ignored.
</p>

</subsubsect>

<subsubsect id="SphericalPolarMatchEngine">
<subhead><title><code>sky3d</code>: Spherical Polar Matching</title></subhead>

<p>&matcher-sky3d-usage;</p>

<p>The <code>sky3d</code> matcher compares positions in the volume of 
the sky taking account of distance from the observer.
The position in three-dimensional space is calculated for each 
row using the <code>ra</code>, <code>dec</code> and <code>distance</code>
as spherical polar coordinates, where <code>distance</code> is the
distance from the observer along the line of sight.
Rows are considered to match when their positions in this space are
within <code>error</code> units of each other.
The units of <code>error</code> are the same as those of 
<code>distance</code>.
</p>

<p>As with <ref id="SkyMatchEngine"><code>sky</code></ref> matching,
other longitude/latitude coordinate pairs may be used in place of
right ascension and declination.
</p>

</subsubsect>

<subsubsect id="EqualsMatchEngine">
<subhead><title><code>exact</code>: Exact Matching</title></subhead>

<p>&matcher-exact-usage;</p>

<p>The <code>exact</code> matcher compares arbitrary key values 
for exact equality.
Rows are considered to match only if the values in their 
<code>matched-value</code> columns are exactly the same.
These values can be strings, numbers, or anything else.
A blank value never matches, not even with another blank one.
Since the <code>params</code> parameter holds no values, 
it does not have to be specified.
Note that the values must also be of the same type, so for instance
a Long (64-bit) integer value will not match an Integer (32-bit) value.
</p>

</subsubsect>

<subsubsect id="IsotropicCartesianMatchEngine">
<subhead><title><code>1d</code>, <code>2d</code>, ...:
         Isotropic Cartesian Matching</title></subhead>

<p>&matcher-1d-usage;</p>

<p>&matcher-2d-usage;</p>

<p>The <code>1d</code> matcher compares positions in 1-dimensional 
Cartesian space.
Rows are considered to match if their <code>x</code> column
values differ by no more than <code>error</code>.
</p>

<p>The <code>2d</code> matcher compares postions in 2-dimensional
Cartesian space.
Rows are considered to match if the difference in their 
(<code>x</code>,<code>y</code>) positions reckoned using
Pythagoras is less than <code>error</code>.
</p>

<p>Matching in any number of Cartesian dimensions can be done by
extending this syntax in the obvious way.
</p>

</subsubsect>

<subsubsect id="AnisotropicCartesianMatchEngine">
<subhead><title><code>2d_anisotropic</code>, ...:
         Anisotropic Cartesian Matching</title></subhead>

<p>&matcher-2d_anis-usage;</p>

<p>The <code>2d_anisotropic</code> matcher compares positions in 
2-dimensional Cartesian space using an anisotropic metric.
Rows are considered to match if their (<code>x</code>,<code>y</code>)
positions fall within an error ellipse with axis lengths 
<code>error-in-x</code>, <code>error-in-y</code> of each other.
This kind of match will typically be used for non-'spatial' spaces,
for instance (magnitude,redshift) space, in which the metrics along
different axes are not related to each other.
</p>

<p>Matching in any number of dimensions of Cartesian space using an
anisotropic metric can be done by extending this syntax in the obvious way.
</p>

</subsubsect>

<subsubsect id="CuboidCartesianMatchEngine">
<subhead><title><code>2d_cuboid</code>, ...:
         Cuboid Cartesian Matching</title></subhead>

<p>&matcher-2d_cuboid-usage;</p>

<p>The <code>2d_cuboid</code> matcher compares positions
in 2-dimensional Cartesian space in cuboidal cells.
Rows are considered to match if their (<code>x</code>,<code>y</code>)
positions fall within an error cuboid with half-axis lengths
<code>error-in-x</code>, <code>error-in-y</code> of each other.
This kind of match is suitable for grouping items into pixels,
though it's not a very efficient way of doing that.
</p>

<p>Matching in any number of dimensions using N-dimensional hyper-cuboids
can be done by extending this syntax in the obvious way.
</p>

</subsubsect>

<subsubsect id="ErrorCartesianMatchEngine">
<subhead><title><code>1d_err</code>, <code>2d_err</code>, ...:
         Cartesian Matching with Per-Object Errors</title></subhead>

<p>&matcher-2d_err-usage;</p>

<p>The <code>1d_err</code>, <code>2d_err</code>, ... matchers compare
positions in N-dimensional Cartesian space like the
<code>1d</code>, <code>2d</code> matchers described in
<ref id="IsotropicCartesianMatchEngine"/>,
except that the match radius can be different for each row.
Rows are considered to match when the separation reckoned by
Pythagoras between the <code>x</code>, <code>y</code>, ... positions 
is no larger than the sum of the two per-row <code>error</code> values.
Matching in any number of Cartesian dimensions can be done by extending
this syntax in the obvious way.
</p>

<p>A variant form <code>1d_err_q</code>, <code>2d_err_q</code>, ...
does the same thing but combines the two per-row <code>error</code> values
in quadrature rather than by summation.
In that case the separation beween the two positions
must be no larger than <m>sqrt(error1<sup>2</sup>+error2<sup>2</sup>)</m>.
</p>

<p>The <code>scale</code> parameter must be supplied, and should be 
approximately the characteristic size of the per-object error values.
In conjunction with the <code>bin-factor</code> tuning parameter
its value affects the performance of the match, but not the result.
</p>

</subsubsect>

<subsubsect id="EllipseCartesianMatchEngine">
<subhead><title><code>2d_ellipse</code>:
         Cartesian Matching of Elliptical Regions</title></subhead>

<p>&matcher-2d_ellipse-usage;</p>

<p>The <code>2d_ellipse</code> matcher compares elliptical regions
in a 2d plane for overlap.  Each row has to specify five values,
giving the centre, the major and minor radii, and the orientation angle
of an ellipse.
Rows are considered to match if there is any overlap between the ellipses.
The goodness of match is a normalised generalisation of the symmetrical case
used by the isotropic matcher, in which the best possible match is 
two concentric ellipses, and the worst allowable match is when 
the circumferences just touch.
</p>

<p>Note the orientation angle is measured anticlockwise from the horizontal,
unlike the position angle used by the 
<ref id="EllipseSkyMatchEngine"><code>skyellipse</code></ref> matcher.
</p>

<p>The <code>scale</code> parameter must be supplied, and should be 
approximately the characteristic size of the per-object major radius.
In conjunction with the <code>bin-factor</code> tuning parameter
its value affects the performance of the match, but not the result.
</p>

</subsubsect>

<subsubsect>
<subhead><title>Custom Matchers</title></subhead>

<p>For advanced users, it is possible to supply the name of a class
on the classpath which implements the 
<code>uk.ac.starlink.table.join.MatchEngine</code> interface
and which has a no-arg constructor.
This allows java programmers to write their own matchers using any
match criteria and binning algorithms they choose.
</p>

</subsubsect>

<subsubsect id="CombinedMatchEngine">
<subhead><title>Matcher Combinations</title></subhead>

<p>In addition to the matching criteria listed in the previous subsections, 
you can build your own by combining any of these.  
To do this, take the two (or more)
matchers that you want to use, and separate their names with a
"<code>+</code>" character.  The <code>values*</code> parameters
of the combined matcher should then hold the concatenation of the
<code>values*</code> entries of the constituent matchers, and the
same for the <code>params</code> parameter.
Two rows are then considered to match if the the match is successful
for all of their constituent matchers.
</p>

<p>A variant form where the names are separated with a
"<code>*</code>" character instead of "<code>+</code>" may also be used.
In this case an additional constraint is applied requiring that
the distance measure (see below) is less than or equal to unity,
thus requiring the points in the notional scaled parameter coordinate space
to be within a unit hypersphere rather than a unit hypercube.
</p>

<p>So for instance the matcher "<code>sky+1d</code>" could be used
with the following syntax:

&matcher-sky.1d-usage;

This would compare positions on the sky with an additional scalar constraint.
Rows are considered to match if <em>both</em>
their <code>ra</code>, <code>dec</code> positions are within
<code>max-error</code> arcseconds of each other along a great circle
(as for <code>matcher=sky</code>) 
<em>and</em>
their <code>x</code> values differ by no more than <code>error</code>
(as for <code>matcher=1d</code>).
Using <code>matcher=sky*1d</code> instead would work the same way
but restrict the matches a bit further.
</p>

<p>This example might be used for instance to identify objects from two
catalogues which are within a couple of arcseconds and also 0.5
blue magnitudes of each other.
Rolling your own matchers in this way can give you quite flexible match
constraints.
</p>

<p>When identifying the closest match
(e.g. <code>find=best1</code> in <code>tmatch2</code>)
the "distance" measure is obtained by
scaling the distances from each of the constituent matchers
and adding these scaled distances in quadrature,
so that each element of the matcher has approximately equal weight.
Scaling is generally done using the maximum permissible match
radius (or equivalent), so the distance measure looks something like
<m>d = sqrt([d<sub>A</sub>/max(d<sub>A</sub>)]<sup>2</sup>
          + [d<sub>B</sub>/max(d<sub>B</sub>)]<sup>2</sup>)</m>.
However the details are a bit dependent on which matchers you are combining.
If the "<code>*</code>" separator is used instead of "<code>+</code>"
in the matcher specification as described above,
this distance will always be &lt;=1 for successful matches.
</p>

<p><em>Note that in STILTS v3.0-9 and earlier,
a linear unscaled distance measure was used here instead,
which did not give very meaningful Best match results.</em>
</p>

</subsubsect>

</subsect>

<subsect id="matchGroup">
<subhead><title>Multi-Object Matches</title></subhead>

<p>The generic matching in STILTS is determined by specified 
match criteria, as described in <ref id="MatchEngine"/>.
These criteria give conditions for whether two items (table rows)
count as matched with each other.  In the case of a pair match,
as provided by <ref id="tmatch2"><code>tmatch2</code></ref>,
it is clear how this is to be interpreted.
</p>

<p>However, some of the matching tasks 
(<ref id="tmatchn"><code>tmatchn</code></ref> in group mode and
 <ref id="tmatch1"><code>tmatch1</code></ref>)
search for match groups which may have more than two members.
This section explains precisely how STILTS applies the pair-wise
matching criteria it is given to identifying multi-object groups.
</p>

<p>In a multi-object match context, the matcher identifies a matched
group as the largest possible group of objects in which each is 
linked by a pair match to <em>any</em> other object in the group -
it is a group of "friends of friends".
Formally, the set of matched groups is a set of disjoint graphs whose nodes
are input table rows and whose edges are successful pair matches,
where no successful pair match exists between nodes in 
different elements of that set.
Thus the set has a minimal number of elements, and each of its elements
is a matched group of maximal size.
The important point to note
is that for any particular pair in a matched group,
there is no guarantee that the two objects match each other,
only that you can hop from one to the other via pairs which do match.
</p>

<p>So in the case of a multi-object sky match
on a field which is very crowded compared to the specified error radius,
it is quite possible for <em>all</em> the objects in the input table(s) 
to end up as part of the same large matching group.  
Results at or near this percolation threshold are
(a) probably not useful and
(b) likely to take a long time to run. 
Some care should therefore be exercised when specifying match criteria
in multi-object match contexts.
</p>

</subsect>

</sect>

<sect id="plot2">
<subhead><title>Plotting</title></subhead>

<p>As of version 3.0 (October 2014), STILTS offers plotting commands
corresponding to the new-style plots in
version 4 of the
<webref url="http://www.starlink.ac.uk/topcat/">TOPCAT</webref> application.
The commands are currently:
<ul>
<li><ref id="plot2plane" plaintextref="yes"><code>plot2plane</code></ref>:
    &plot2plane-purpose;
    </li>
<li><ref id="plot2sky" plaintextref="yes"><code>plot2sky</code></ref>:
    &plot2sky-purpose;
    </li>
<li><ref id="plot2cube" plaintextref="yes"><code>plot2cube</code></ref>:
    &plot2cube-purpose;
    </li>
<li><ref id="plot2sphere" plaintextref="yes"><code>plot2sphere</code></ref>:
    &plot2sphere-purpose;
    </li>
<li><ref id="plot2corner" plaintextref="yes"><code>plot2corner</code></ref>:
    &plot2corner-purpose;
    </li>
<li><ref id="plot2time" plaintextref="yes"><code>plot2time</code></ref>:
    &plot2time-purpose;
    </li>
</ul>
(In previous versions the less capable commands
<code>plot2d</code>,
<code>plot3d</code> and
<code>plothist</code>
were available -
these are now deprecated, but described in <ref id="plot"/>).
</p>

<p>These commands all have a similar structure.
The <em>plot surface</em>, or geometry of the plot,
is defined by which command you use
(for instance, if you want to plot longitude/latitude data on
the celestial sphere, use <code>plot2sky</code>).
Content is added to the plot by specifying zero or more
<em>plot layers</em>,
as described in <ref id="LayerType"/> below.
<ref id="ShapeMode"/> describes the shading modes which affect
how colouring is performed for some of the layer types.
Once a plot has been specified, it can be displayed on
the screen or exported in some way according to
a selected output mode (<ref id="paintMode"/>)
and perhaps export format (<ref id="graphicExporter"/>).
Plots displayed to the screen are by default "live" - they can be
resized and navigated around (pan, zoom, rotate, ...) using the mouse
in the same way as in a TOPCAT window.
</p>

<p>These commands allow you to make all the plots that can be produced
with TOPCAT, in some cases with more flexibility in configuration.
Unlike TOPCAT, the size of table you can plot is not limited by
the size of table you can load into the application.
In most cases, STILTS will generate plots from arbitrarily large
data sets with fixed (and modest) memory requirements.
Performance is of course highly dependent on the details of the plot,
but for instance an all-sky density plot for 2 billion points can
be produced in the order of 30 minutes.
</p>

<subsect id="TypedPlot2Task">
<subhead><title>Plot Parameters</title></subhead>

<p>The plotting commands offer a great deal of control over what is plotted
and how it is represented, and thus unavoidably have lots of parameters.
When looking at the command documentation in <ref id="cmdUsage"/>
the <ref id="plot2plane-usage">Usage</ref> sections may look rather daunting.
However, the discussion below and the 
<ref id="plot2plane-examples">Examples</ref> sections should help.
Generating a simple plot is straightforward and can be done with only
four or five parameters; if you want to represent more complicated data
or have specific preferences for
appearance then you can consult the documentation for the additional options.
</p>

<p>As a simple example, 
if a file "cat.fits" contains the columns RMAG and BMAG for red and 
blue magnitudes,
you can draw a two-dimensional colour-magnitude scatter plot with
the command:
<verbatim>
   stilts plot2plane layer_1=mark in_1=cat.fits x_1=BMAG-RMAG y_1=BMAG
</verbatim>
Since an output file is not specified, the plot is shown in a window
on the screen.
This plot window is "live" - you can resize the window,
or pan and zoom around it using the same mouse controls as in TOPCAT.
To send the output to a PNG file, do instead:
<verbatim>
   stilts plot2plane layer_1=mark in_1=cat.fits x_1=BMAG-RMAG y_1=BMAG out=fig.png
</verbatim>
</p>

<p>We can adjust the plot by inverting the Y axis so it
increases downwards instead of upwards:
<verbatim>
   stilts plot2plane
              yflip=true
              layer_1=mark in_1=cat.fits x_1=BMAG-RMAG y_1=BMAG
</verbatim>
The parameters of the plot now fall into two groups.
Global parameters, without suffixes,
make global adjustments to the plot.
In this example <code>yflip=true</code> inverts the Y axis.
<strong>Layer parameters</strong>, with suffixes,
are introduced by a <code>layer</code> parameter
and grouped together by a given suffix.
Each layer group defines a plot layer with content
to be drawn on the plot surface.
In this case the layer is of type <code>mark</code> (draw markers)
and the suffix is "<code>_1</code>".
Global and Layer parameters are described separately in the following
subsections.
</p>

<subsubsect id="plot2-global-params">
<subhead><title>Global Parameters</title></subhead>

<p>The global plot parameters are documented in
the usage sections of the various plot commands
(e.g. <ref id="plot2plane-usage"/>).
They deal with things like positioning the plot axes,
fixing the data bounds, selecting font types and sizes,
adjusting grids and tickmarks, configuring how interactive
navigation works, managing data storage, and so on.
They are all optional,
since they all have sensible defaults, for instance data bounds
will be determined from the supplied data if they are not given
explicitly.
</p>

</subsubsect>

<subsubsect id="plot2-layer-params">
<subhead><title>Layer Parameters</title></subhead>

<p>The layer parameters come in groups, each specifying the
details of one plot layer.
Each layer type has its own list of parameters.
A plot layer is introduced on the command line with a parameter
of the form
<verbatim>
   layer&lt;suffix&gt;=&lt;layer-type&gt;
</verbatim>
and any other parameters with the same <code>&lt;suffix&gt;</code>
are considered to apply to the same layer.
In the basic example we considered:
<verbatim>
   stilts plot2plane layer_1=mark in_1=cat.fits x_1=BMAG-RMAG y_1=BMAG
</verbatim>
the suffix is "<code>_1</code>" and the layer type associated with it is 
<code>mark</code> (plotting markers to make a scatter plot).
The different layer types are documented in <ref id="LayerType"/>,
and each has its own set of parameters, some of which are mandatory
and some which are optional with sensible defaults.
In the documentation, the suffix is represented as "<code>N</code>".
For instance the <code><ref id="layer-mark">mark</ref></code> layer type
requires you to specify an input table (<code>inN</code>) and
point positions (<code>xN</code> and <code>yN</code>).
Since the suffix we have used in the example for the
<code>layerN</code> parameter is "<code>_1</code>",
we have written <code>in_1</code>, <code>x_1</code> and <code>y_1</code>.
The <code>mark</code> layer has some optional style parameters as well,
so we could adjust the plot's appearance by adding
<code>shape_1=cross size_1=4 color_1=blue</code>.
</p>

<p>You can have as many layers as you like (even none),
so we could overplot two datasets from different input files like this:
<verbatim>
   stilts plot2plane
       layer_1=mark in_1=cat1.fits x_1=BMAG-RMAG   y_1=BMAG  color_1=magenta size_1=5
       layer_2=mark in_2=cat2.fits x_2=mag_b-mag_r y_2=mag_b color_2=cyan    size_2=5
</verbatim>
We have assigned different colours to the different layers
and boosted the marker size to 5 pixels.
</p>

<p>As a convenience, if the same value is used for all the layers,
you can omit the suffix.  So to avoid having to specify the same
markers size for both layers, you can write instead:
<verbatim>
   stilts plot2plane
       size=5
       layer_1=mark in_1=cat1.fits x_1=BMAG-RMAG   y_1=BMAG  color_1=magenta
       layer_2=mark in_2=cat2.fits x_2=mag_b-mag_r y_2=mag_b color_2=teal
</verbatim>
Although the <code>size</code> parameter no longer has an explicit suffix,
it's still a layer parameter, it just applies to multiple layers.
This shorthand works for all layer parameters.
Here is another example which also shows how you can use the
<code>icmdN</code> parameter to <ref id="pipes">pre-process</ref>
input data prior to performing the plot.
Here, we make two different selections of the input rows to plot
two different data sets.
<verbatim>
   stilts plot2plane
          in=cat.fits x=BMAG-RMAG y=BMAG
          layer_1=mark icmd_1='select vel&lt;1000'  color_1=blue
          layer_2=mark icmd_2='select vel&gt;=1000' color_2=red
</verbatim>
The input tables and data values are the same for both datasets,
so we can just supply the parameters
<code>in</code>, <code>x</code> and <code>y</code>,
rather than
<code>in_1</code>, <code>in_2</code> etc.
</p>

<p>Any string can be used as a suffix, including the empty string
(though an empty string can cause confusion if there are multiple layers).
The suffixing is also slightly more sophisticated than described above;
to find parameters relating to a layer with a given suffix, the
parameter looks first using the whole suffix, and strips single
characters off it until it has none left.
So if a layer is introduced with the parameter <code>layer_ab</code>,
you can give the marker shape using any of the parameters
<code>shape_ab</code>, <code>shape_a</code>, <code>shape_</code>
or <code>shape</code>.  If more than one of these is present,
the first one in that list will be used
(the order in which they appear on the command line is not significant).
This can be used to group sets of layers.
</p>

<p>By default, if multiple layers are specified, they are plotted
in the order in which the introducing <code>layerN</code> parameters
appear on the command line.  This may be relevant, since layers
plotted later sometimes obscure ones plotted earlier.
You can alter the order of plotting with the <code>seq</code> (global)
parameter, which is a comma-separated list of layer suffixes giving
the sequence in which layers should be plotted.
So adding "<code>seq=_2,_1</code>" would cause layer _2 to be plotted
before layer _1, instead of the other way round.
</p>

<p>By default, if more than one layer is plotted, a legend will
appear labelling the datasets.
The dataset labels appearing in the legend are by default
the layer suffixes specified on the command line.
However, the labels can be given explicitly with the <code>legendN</code>
parameter, so for instance in the example above
<code>leglabel_1=Slow leglabel_2=Fast</code> would adjust the
legend accordingly.
Legend appearance and positioning can be adjusted by various
<code>leg*</code> global parameters.
</p>

</subsubsect>

<subsubsect id="animate">
<subhead><title>Animation</title></subhead>

<p>The plotting commands can be used to produce animations.
This is done by supplying an <em>animation control table</em>
using the <code>animate</code> parameter
(which has associated <code>afmt</code> and <code>acmd</code> parameters
for specifying its file format and applying filters).
One output image is produced for each row of the control table.
The columns of the table have names which correspond to plot command
parameters, and for each row, the basic plot command is executed
with the parameters on the command line supplied or replaced by
those from the table.
This is most commonly used for providing a movie of the kind of
navigation you can do interactively with the mouse,
but other applications are possible.
</p>

<p>For instance, given the following animation control table
with the name "bounds.txt", in ASCII format:
<verbatim>
  #  xmax  ymax
      4.0   2.0
      3.0   1.5
      2.0   1.0
      1.0   0.5
</verbatim>
then this command:
<verbatim>
   stilts plot2plane xmin=0 ymin=0
                     layer_1=mark in_1=gums_smc.fits x_1=ag y_1=av
                     animate=bounds.txt afmt=ascii
</verbatim>
would produce a 4-frame animation zooming in towards the origin.
</p>

<p>If output is to the screen
(<code>omode=<ref id="paintmode-swing">swing</ref></code>, the default)
the animation can be seen directly.
If it is to an output file
(<code>omode=<ref id="paintmode-out">out</ref></code>)
then a number of output files is written with sequence numbers,
so adding the parameter "<code>out=x.png</code>" to the above command
would produce 4 files,
<code>x-1.png</code>,
<code>x-2.png</code>,
<code>x-3.png</code> and
<code>x-4.png</code>.
Padding zeros are used to keep the files in alphanumeric sequence,
so for instance in a 500-frame animation the first one would be named
<code>x-001.png</code>.
STILTS does not actually turn these files into a single animated output file,
but you can use other tools to do this, for instance using ImageMagick:
<verbatim>
   convert x-*.png xmovie.gif
</verbatim>
or ffmpeg:
<verbatim>
   ffmpeg -i 'x-%03d.png' -framerate 15 -pix_fmt yuv420p xmovie.webm
</verbatim>
</p>

<p>You can create the animation control table any way you like,
but you may find the <code><ref id="tloop">tloop</ref></code>
command convenient.  For instance the above table can be
written like this:
<verbatim>
   stilts tloop xmax 4 0 -1 ocmd='addcol ymax xmax*0.5' ofmt=ascii out=bounds.txt
</verbatim>
Alternatively, you can generate a table like this inline using the
<ref id="scheme-loop">loop scheme</ref>.
A common requirement is to produce an animation of rotating a 3-d plot,
here's an example of how to do that:
<verbatim>
   stilts plot2sphere layer_1=mark in_1=hip_main.fits lon_1=radeg lat_1=dedeg r_1=plx \
                      animate=:loop:15,375,2 acmd='colmeta -name phi $1'
</verbatim>
The <code>phi</code> parameter controls the angle from which the
3D plot is viewed, and here it is incremented by 2 degrees
for each frame.
The same thing would work for <code>plot2cube</code>
as well as <code>plot2sphere</code>.
</p>

<p>Note that producing animations in this way is usually much more
efficient than writing a shell script which invokes STILTS multiple times.
The plot commands also employ multi-threading when animating to
output files, so should make efficient use of multi-core machines
(though currently animations to the screen are not multi-threaded).
</p>

</subsubsect>

</subsect>

<subsect id="PlotType">
<subhead><title>Surface Types</title></subhead>

<p>The different <code>plot2*</code> commands correspond to
different plot surface geometries.
The different commands come with their own specific
axis configuration parameters.
Some of the plot layer types are specific to certain surface types.
When supplying data from input tables to plot layers,
the coordinate values you need to supply
(and hence the corresponding parameter names)
are determined not by the layer type, but by the surface type.
For instance, point positions for layer N on
a 2-d Cartesian surface (<code>plot2plane</code> command)
are given using parameters <code>xN</code> and <code>yN</code>,
but when plotting to the celestial sphere
(<code>plot2sky</code> command) you supply
<code>lonN</code> and <code>latN</code>).
</p>

<p>The following list summarises the available surface types and
their corresponding positional coordinates.

<dl>
<dt>Plane (<code><ref id="plot2plane">plot2plane</ref></code>)</dt>
<dd><p>2-dimensional Cartesian axes.
    Positional coordinates are supplied as
    <code>x</code>, <code>y</code> pairs.
    Note that this command can also be used to draw histograms.
    </p></dd>

<dt>Sky (<code><ref id="plot2sky">plot2sky</ref></code>)</dt>
<dd><p>Celestial sphere.
    Positional coordinates are supplied as
    <code>lon</code>, <code>lat</code> pairs,
    giving longitude and latitude in decimal degrees.
    A number of different projections are available,
    and conversion between different celestial coordinate systems can
    also be performed.
    You could use it for other spherical coordinate systems too
    (like the surface of a planet).
    </p></dd>

<dt>Cube (<code><ref id="plot2cube">plot2cube</ref></code>)</dt>
<dd><p>3-dimensional Cartesian axes.
    Positional coordinates are supplied as
    <code>x</code>, <code>y</code>, <code>z</code> triples.
    </p></dd>

<dt>Sphere (<code><ref id="plot2sphere">plot2sphere</ref></code>)</dt>
<dd><p>3-dimensional isotropic space with spherical polar coordinates.
    Positional coordinates are supplied as
    <code>lon</code>, <code>lat</code>, <code>r</code> triples,
    giving longitude and latitude in decimal degrees, and radius in
    an arbitrary unit.
    The plotting surface (space) is similar to Cube,
    except that the unit distance is always
    the same in all three directions.
    </p></dd>

<dt>Matrix (<code><ref id="plot2corner">plot2corner</ref></code>)</dt>
<dd><p>Grid of scatter plots, one for each pair of N coordinates.
    Positional coordinates are supplied as
    <code>x1</code>, <code>x2</code>, <code>x3</code>, ... <code>xN</code>,
    up to the number N specified by the <code>nvar</code> parameter.
    </p></dd>

<dt>Time (<code><ref id="plot2time">plot2time</ref></code>)</dt>
<dd><p>2-dimensional axes, but the horizontal axis represents time.
    The axis may be labelled in various ways
    (ISO-8601 dates, decimal year, MJD etc). 
    Positional coordinates are supplied as
    <code>t</code>, <code>y</code> pairs.
    Time can be represented in input data in various ways;
    if sufficient metadata is provided in the input format
    the epoch can be determined automatically,
    otherwise it may be necessary to specify the time representation
    being used.
    </p></dd>
</dl>
</p>

</subsect>

<subsect id="LayerType">
<subhead><title>Layer Types</title></subhead>

<p>The different plot layers and how to configure them with parameters
is given in the following subsections.
The layers which may be plotted on a particular surface depend on
the plot geometry,
so not all of these are available for every plot command.
</p>

&layertype-docs;

</subsect>

<subsect id="ShapeMode">
<subhead><title>Shading Modes</title></subhead>

<p>Some plot <ref id="LayerType">layer types</ref>
have an associated <code>shading</code> parameter
which determines how plotted markers are coloured.
This is independent of the marker shapes
(which may be points, vectors, ellipses, ...)
but may be affected by how many markers are plotted on top of each other,
additional input table values, selected colour maps etc.
For the simplest shading types
(e.g. <ref id="shading-flat"><code>flat</code></ref>)
it's just a case of choosing a colour,
but the more complex ones have several associated parameters.
</p>

<p>The various shading types and their usages are described in
the following subsections.
</p>

&shapemode-docs;

</subsect>


<subsect id="paintMode">
<subhead><title>Output Modes</title></subhead>

<p>The plots generated by the plotting commands can be used in various
different ways.  One thing you might want to do is to write the
output to a file in a given graphics format 
(<ref id="paintmode-out"><code>out</code></ref>);
another is to preview it directly on the screen
(<ref id="paintmode-swing"><code>swing</code></ref>).
By default one or other of these will happen depending on whether you
specify an output file.  However there are other possibilities;
these are listed in the following subsections.
</p>

<p>Except for display to the screen, these modes should work happily
on a headless machine (one with no graphics display, as may be the case
for a web server).
When running headless, you may find it necessary to 
set the java system property
"<code>java.awt.headless</code>" to <code>true</code>
- see <ref id="sysProperties"/>.
</p>

<p>The default output mode is <ref id="paintmode-auto"><code>auto</code></ref>,
which means that output is to a file if an output file is specified,
or to the screen if it is not.  So in most cases you don't need to specify the
<code>omode</code> parameter explicitly.
</p>

&paintmode-docs;

</subsect>


<subsect id="graphicExporter">
<subhead><title>Export Formats</title></subhead>

<p>Several of the plot output <ref id="paintMode">modes</ref>
write the plot in some graphics format or other.
When selecting an output format it is important to understand the 
distinction between <em>bitmapped</em> and <em>vector</em> formats;
basically bitmapped formats represent the image as a grid of finite-sized
pixels while vector formats notionally draw smooth lines.
Bitmapped formats are fine for a computer screen, but for high quality
paper printouts you will want a vector format.
You can convert from vector to bitmapped but not (usefully) in the
other direction.  There are a couple of subtleties to this distinction
specific to STILTS graphical output as discussed below.
</p>

<p>The following formats are the available values for the
<code>ofmt</code> parameter of the various plot commands:
<dl>
<dt><code>png</code></dt>
<dd><p>PNG format.
    This is a flexible bitmapped format providing transparency 
    and an unlimited number of colours with good lossless compression.
    It is widely supported by non-ancient browsers and other image viewers,
    and is generally recommended for bitmapped output.
    </p></dd>

<dt><code>gif</code></dt>
<dd><p>GIF format.
    This is a bitmapped format providing transparency and lossless
    compression.
    The number of colours is limited to 255 however, so if you are using
    auxiliary axes (colour variation to represent higher dimensionality)
    or other plot features which use a wide range of colours you may see
    image degradation.
    It has long been widely supported by browsers and other image viewers.
    </p></dd>

<dt><code>jpeg</code></dt>
<dd><p>JPEG format.
    This is a bitmapped format with lossy compression
    intended primarily for photographs.
    Transparency is not supported, and although there is no limit on the
    maximum number of colours, its lossiness means that plots generated
    using it generally look a bit smudged.
    </p></dd>

<dt><code>pdf</code></dt>
<dd><p>Portable Document Format.
    This is the format which can be read by Adobe's Acrobat Reader.
    It is a widely portable vector format, and is suitable for printing
    at high resolution, either standalone or imported into some other 
    presentation format.
    However, there are a couple of caveats when it comes to using it with
    STILTS plots.
    <ol>
    <li>If used to plot a very large number of points, the output PDF file
        can get quite large, though it's much better than for
        <code>eps</code> output (see below).
        </li>
    <li>For certain colour shading options
        (<ref id="shading-auto">auto</ref>,
         <ref id="shading-density">density</ref>,
         and in some circumstances transparency),
        the body of the plot will
        be drawn as a bitmap rather than vector graphics.
        This is sometimes a blessing in disguise since with very large numbers
        of points a vector PDF file could get unmanageably large in any case.
        In this case the interior of the plot will be pixellated.
        The axes and annotations outside of the
        plot will still be drawn in vector format however.
        </li>
    </ol>
    </p></dd>

<dt><code>svg</code></dt>
<dd><p>Scalable Vector Graphics.
    This is an XML-based vector graphics format developed for display
    in web pages, and
    <webref url="https://www.w3.org/TR/SVG/">defined by the W3C</webref>.
    This exporter can generate
    OutOfMemoryErrors if asked to generate a large output file.
    </p></dd>

<dt><code>eps</code></dt>
<dd><p>Encapsulated Postscript.
    This is a vector format which is suitable for printing at high resolution
    either standalone or imported into some other presentation format
    (you may need to convert it via PDF depending on the intended destination).
    However, there are a couple of caveats when it comes to using it with
    STILTS plots.
    <ol>
    <li>Unfortunately the postscript driver used by STILTS is not very 
        efficient and can result in large, sometimes very large, postscript
        output files.  This is likely to be a problem for plots with a large
        number of non-transparent points.
        For this reason <code>eps-gzip</code> or <code>pdf</code> may be
        a better choice.
        </li>
    <li>Postscript has no support for partial transparency, so if plots 
        are drawn with partially transparent points (common for very large
        data sets) the only way they can be rendered is by drawing the body
        of the plot as a bitmap rather than as vector graphics.
        This is sometimes a blessing in disguise since with very large numbers
        of points a vector postscript file would likely be unmanageably 
        large in any case.
        So if there is any transparency in the plot, the interior of the
        plot will be pixellated.  The axes and annotations outside of the
        plot will still be drawn in vector format however.
        </li>
    </ol>
    </p></dd>

<dt><code>eps-gzip</code></dt>
<dd><p>Just like the <code>eps</code> format above except that the output
    is automatically compressed using the GZIP format as it is written.
    Postscript compresses well (typically a factor of 5-10).
    </p></dd>

</dl>
</p>

</subsect>

<subsect id="Shaders">
<subhead><title>Colour Maps</title></subhead>

<p>A number of colour maps are available, and used for instance
with the <ref id="ShapeMode">shading modes</ref>
<ref id="shading-density"><code>density</code></ref>,
<ref id="shading-aux"><code>aux</code></ref> and
<ref id="shading-weighted"><code>weighted</code></ref>.
Not all colour maps are suitable/available in all contexts,
and in some cases the maps are by default clipped at one end to avoid
for instance white-on-white plotting,
but the lists below give an overview of which named colourmaps can be used.
Note that custom colourmaps can also be used by supplying a specification
like "<code>red-yellow-blue</code>".
</p>

<p>The <em>absolute</em> colour maps are listed below:
these do not depend on the underlying colour of the plotted symbols,
so are suitable when only one dataset is being plotted.
</p>

<figure>
<caption><px>Absolute colour maps</px></caption>
<figureimage src="&FIG.colormaps-abs;"/>
</figure>

<p>The <em>non-absolute</em> colour maps are listed below:
these modify an underlying colour, so are suitable for applying
to several different datasets with different underlying colours.
The representation here shows how they affect several different colours; 
for each row of pixels the unmodified (value=0) colour is at the left of
the image and the most modified (value=1) is at the right.
</p>

<figure>
<caption><px>Non-absolute colour maps</px></caption>
<figureimage src="&FIG.colormaps-noabs;"/>
</figure>

<p>These colour maps have been derived from several sources, including
SkyCat/GAIA,
MatPlotLib 1.5,
Gnuplot, Daniel Michalik,
<webref url="https://personal.sron.nl/~pault/">Paul Tol</webref>,
<webref url="https://github.com/1313e/CMasher">CMasher</webref>,
<webref url="http://colorbrewer2.org/">Color Brewer</webref>,
<webref url="http://hclwizard.org/">HCL Wizard</webref>,
<webref url="https://www.mrao.cam.ac.uk/~dag/CUBEHELIX/">Dave Green</webref>,
<webref url="https://xkcd.com/2537/">xkcd</webref>,
and maybe some others I forgot.
</p>

</subsect>

</sect>

<sect id="plot">
<subhead><title>Old-Style Plotting</title></subhead>

<p><em>This section describes deprecated commands.
For recommended plotting commands, see <ref id="plot2"/>.
</em></p>

<p>From version 2.0 (October 2008), STILTS incorporated three table
plotting commands:
<ul>
<li><ref id="plot2d"><code>plot2d</code></ref>: &plot2d-purpose;</li>
<li><ref id="plot3d"><code>plot3d</code></ref>: &plot3d-purpose;</li>
<li><ref id="plothist"><code>plothist</code></ref>: &plothist-purpose;</li>
</ul>
These provided command-line access to some, though not all,
of the plotting capabilities offered by TOPCAT.
</p>

<p>Since version 3.0 (October 2014), these commands are deprecated
in favour of the more powerful ones described in <ref id="plot2"/>.
The rest of this section describes some aspects of the deprecated
commands for the benefit of legacy code.
The output modes and formats are the same in old- and new-style plots,
and are discussed in <ref id="paintMode"/> and <ref id="graphicExporter"/>.
The handling of parameters and suffixes for these commands
is not quite the same as for new-style plots,
and is documented in the next subsection.
</p>

<p>As a simple example, 
if a file "cat.fits" contains the columns RMAG and BMAG for red and 
blue magnitudes,
you can draw a two-dimensional colour-magnitude scatter plot with
the command:
<verbatim>
   stilts plot2d in=cat.fits xdata=BMAG-RMAG ydata=BMAG
</verbatim>
Since an output file is not specified, the plot is shown on the screen for
convenience.  To send the output to a PNG file, do instead:
<verbatim>
   stilts plot2d in=cat.fits xdata=BMAG-RMAG ydata=BMAG out=plot.png ofmt=png
</verbatim>
in some cases (including the above), the <code>ofmt</code> parameter is not
required since STILTS may be able to guess the format from the output
file name.
Various other options for output and graphics formats are described in 
<ref id="paintMode"/> and <ref id="graphicExporter"/>
</p>

<p>Some of the parameters use suffixes to define data sets and 
therefore behave a bit differently
from the parameters elsewhere in STILTS - a discussion of these
is given in <ref id="plotSuffixes">the following subsection</ref>.
Some other plotting-specific topics are also discussed below.
</p>

<subsect id="plotSuffixes">
<subhead><title>Parameter Suffixes</title></subhead>

<p><em>This section describes deprecated commands.
For recommended plotting commands, see <ref id="plot2"/>.
</em></p>

<p>Some of the parameters for the plotting tasks behave a little bit
differently to other parameters in STILTS, in order to accommodate
related sets of values.  If you look at the usage of one of the 
plotting commands, for instance in <ref id="plot2d-usage"/>,
you will see that a number of the parameters have the 
suffixes "<code>N</code>" or "<code>NS</code>".
These suffixes can be substituted with any convenient string to identify 
parameters which relate to the same input datasets or subsets.
Specifically:
<dl>
<dt>Suffix "<code>N</code>":</dt>
<dd><p>Denotes an input dataset.  At least the <code>inN</code> parameter
    must be given to identify the source of the data; any other parameters
    with the same value of the <code>N</code> suffix relate to that dataset.
    A <em>dataset</em> here refers to a particular set of plot data from a
    table; in most cases each input table corresponds to a different dataset,
    though two datasets may correspond to different sets of columns from
    the same table.
    </p></dd>
<dt>Suffix "<code>NS</code>":</dt>
<dd><p>Denotes a particular subset of the rows in dataset <code>N</code>.
    At least the <code>subsetNS</code> parameter must be given to 
    identify the expression by which the subset is defined;
    any other parameters with the same value of the <code>NS</code> suffix
    relate to that subset.
    </p></dd>
</dl>
</p>

<p>Some examples will help to illustrate.
The following will generate a Cartesian plot of catalogue position
from a single dataset:
<verbatim>
   stilts plot2d in=gals.fits xdata=RA ydata=DEC
</verbatim>
In this case the <code>N</code> suffix is present on each of the parameters
<code>in</code>, <code>xdata</code> and <code>ydata</code>, but is
equal to the empty string, hence invisible.  This is perfectly legal,
and convenient when only a single table is in use.
If we wish to overplot two datasets however, the dataset suffixes 
(or one of them at least) have to be made explicit so that different ones 
can be used, for instance:
<verbatim>
   stilts plot2d in1=gals.fits  xdata1=RA      ydata1=DEC
                 in2=stars.fits xdata2=RAJ2000 ydata2=DEJ2000
</verbatim>
The suffix values "<code>1</code>" and "<code>2</code>" are quite arbitrary
and can be chosen as convenient, so the following would do exactly the
same as the previous example:
<verbatim>
   stilts plot2d in_GAL=gals.fits   xdata_GAL=RA       ydata_GAL=DEC
                 in_STAR=stars.fits xdata_STAR=RAJ2000 ydata_STAR=DEJ2000
</verbatim>
The other parameters which have the <code>N</code> suffix apply only
to the matching dataset, so for instance the following:
<verbatim>
   stilts plot2d in1=gals.fits  xdata1=RA      ydata1=DEC     txtlabel1=NGC_ID
                 in2=stars.fits xdata2=RAJ2000 ydata2=DEJ2000
</verbatim>
would draw text labels adjacent to the points from only the gals.fits file
giving the contents of its NGC_ID column.
</p>

<p>The <code>NS</code> suffix identifies distinct <em>row subsets</em> 
within the same or different datasets.  A subset is defined by supplying
a boolean inclusion expression (each row is included only if the
expression evaluates true for that row) as the value of a 
<code>subsetNS</code> parameter.
If, as in all the examples we have seen so far, 
no <code>subsetNS</code> parameter is supplied for a given dataset,
then it is treated as a special case, as if a single subset with a name
equal to the empty string (<code>S=""</code>) 
containing all rows has been specified.  
So our earlier simple example:
<verbatim>
   stilts plot2d in=gals.fits xdata=RA ydata=DEC
</verbatim>
is equivalent to
<verbatim>
   stilts plot2d in=gals.fits xdata=RA ydata=DEC subset=true
</verbatim>
If we wish to split the plotted points into two sets based on their 
R-B colours, we can write something like:
<verbatim>
   stilts plot2d in=gals.fits xdata=RA ydata=DEC
                 subsetX='RMAG-BMAG&gt;0' subsetY='RMAG-BMAG&lt;=0'
</verbatim>
This will generate a plot with two subsets shown using different colours
and/or plotting symbols.  These colours and symbols
are selected automatically.  More control over the appearance can be
exercised by setting values for some of the other parameters with
<code>NS</code> suffixes, for instance
<verbatim>
   stilts plot2d in=gals.fits xdata=RA ydata=DEC
                              subset_A='RMAG-BMAG&gt;0'  colour_A=blue
                              subset_B='RMAG-BMAG&lt;=0' colour_B=red
</verbatim>
Again, the suffix strings can be chosen to have any value as convenient.
</p>

<p>The dataset- and subset-specific parameters must be put
together if there are multiple datasets with multiple subsets to plot
simultaneously, for instance:
<verbatim>
   stilts plot2d in_1=gals.fits  xdata_1=RA ydata_1=DEC
                                 subset_1_A='RMAG-BMAG&gt;0'  colour_1_A=blue
                                 subset_1_B='RMAG-BMAG&lt;=0' colour_1_B=red
                 in_2=stars.fits xdata_2=RAJ2000 ydata_2=DEJ2000
                                 colour_2=green
</verbatim>
</p>

<p>Finally, it's not quite true that the suffixes chosen have no effect
on the plot; they may influence the order in which sets are plotted.
Markers drawn for sets plotted earlier may be obscured by the markers
drawn for sets plotted later, so this can affect the appearance of the plot.
If you want to control this, use the <code>sequence</code> parameter.
For instance, to ensure that star data appears on top of galaxy data 
in the plot, do the following:
<verbatim>
   stilts plot2d in_GAL=gals.fits   xdata_GAL=RA       ydata_GAL=DEC
                 in_STAR=stars.fits xdata_STAR=RAJ2000 ydata_STAR=DEJ2000
                 sequence=_GAL,_STAR
</verbatim>
</p>

<p>More examples can be found in the <label>Examples</label>
subsections of the individual plotting command descriptions in
<ref id="cmdUsage"/>.
</p>

</subsect>

</sect>


<sect id="jel">
<subhead><title>Algebraic Expression Syntax</title></subhead>

<p>Many of the STILTS commands 
allow you to use algebraic expressions based on table columns when doing
things like making row selections, defining new columns, selecting values
to plot or match, and so on.
In these cases you are defining an expression which
has a value in each row as a function of the values in the existing
columns in that row.
This is a powerful feature which permits you to manipulate and select
table data in very flexible ways.
The syntax for entering these expressions is explained in this section.
</p>

<p>What you write are actually expressions in
the Java language, which are compiled into Java bytecode before
evaluation.  However, this does not mean that you need to be a
Java programmer to write them.  The syntax is pretty similar to C,
but even if you've never programmed in C most simple things,
and many complicated ones, are quite intutitive.
</p>

<p>The following explanation gives
some guidance and <ref id="jelExamples">examples</ref>
for writing these expressions.
Unfortunately a complete tutorial on writing Java expressions is beyond
the scope of this document, but it should provide enough information
for even a novice to write useful expressions.
</p>

<p>The expressions that you can write are basically any function
of all the column values which apply
to a given row; the function result can then be used where STILTS
needs a per-row value, for instance to define a new column.
If the built-in operators and functions are not sufficient,
or it's unwieldy to express your function in one line of code,
it is possible to add new functions by writing your own classes -
see <ref id="jelExtend"/>.
</p>

<p>Note that since these algebraic expressions often contain spaces,
you may need to enclose them in single or double quotes so that
they don't get confused with other parts of the command string.
</p>

<p><strong>Note:</strong> if Java is running in an environment with
certain security restrictions (a security manager which
does not permit creation of custom class loaders) then algebraic
expressions won't work at all.  It's not particularly likely
that security restrictions will be in place if you are running
from the command line though.
</p>

<subsect id="jel-colref">
<subhead><title>Referencing Column Values</title></subhead>

<p>To create a useful expression which can be evaluated for each row
in a table, you will have to refer to cells in different columns of that row.
You can do this in several ways:
<dl>
<dt>By Name</dt>
<dd><p>The Name of the column may be used if it is unique (no other column in
    the table has the same name) and if it has a suitable form.
    This means that it must have the form of a Java variable - basically
    starting with a letter and continuing with
    letters, numbers, underscores and currency symbols.
    In particular it cannot contain spaces, commas, parentheses etc.
    </p>
    <p>As a special case, if an expression contains just a single column name,
    rather than some more complicated expression, 
    then any column name may be used, even one containing non-alphanumeric
    characters.
    </p>
    <p>Column names are treated case-insensitively.
    </p></dd>

<dt>By &column.id.char;ID</dt>
<dd><p>The "&column.id.char;ID"
    identifier of the column may always be used to refer to it;
    this is a useful fallback if the column name isn't suitable for
    some reason (for instance it contains spaces or is not unique).
    This is just a "&column.id.char;" sign followed by the column index -
    the first column is &column.id.char;1.
    </p></dd>

<dt>By &ucd.prefix; specifier</dt>
<dd><p>If the column has a
    <webref url="http://www.ivoa.net/Documents/latest/UCD.html"
            >Unified Content Descriptor</webref>
    (this will usually only be the case for VOTable or possibly FITS format
    tables) you can refer to it using an identifier of the form
    "<code>&ucd.prefix;&lt;ucd-spec&gt;</code>".  Depending on the version of
    UCD scheme used, UCDs can contain various punctuation marks such
    as underscores, semicolons and dots; for the purpose of this syntax
    these should all be represented as underscores ("<code>_</code>").
    So to identify a column which has the UCD "<code>phot.mag;em.opt.R</code>",
    you should use the identifier "<code>&ucd.prefix;phot_mag_em_opt_r</code>".
    Matching is not case-sensitive.  Futhermore, a trailing underscore
    acts as a wildcard, so that the above column could also be referenced
    using the identifier "<code>&ucd.prefix;phot_mag_</code>".  If multiple
    columns have UCDs which match the given identifer, the first one
    will be used.
    </p>
    <p>Note that the same syntax can be used for referencing table
    parameters (see the <ref id="jel-paramref">next section</ref>);
    columns take preference so if a column and a parameter both match
    the requested UCD, the column value will be used.
    </p></dd>

<dt>By &utype.prefix; specifier</dt>
<dd><p>If the column has a <label>Utype</label>
    (this will usually only be the case for VOTable or possibly FITS format
    tables) you can refer to it using an identifier of the form
    "<code>&utype.prefix;&lt;utype-spec&gt;</code>".
    Utypes can contain various punctuation marks such as colons and dots;
    for the purpose of this syntax
    these should all be represented as underscores ("<code>_</code>").
    So to identify a column which has the Utype 
    "<code>ssa:Access.Format</code>",
    you should use the identifier 
    "<code>&utype.prefix;ssa_Access_Format</code>".
    Matching is not case-sensitive.
    If multiple columns have Utypes which match the given identifier,
    the first one will be used.
    </p>
    <p>Note that the same syntax can be used for referencing table
    parameters (see the <ref id="jel-paramref">next section</ref>);
    columns take preference so if a column and a parameter both match
    the requested Utype, the column value will be used.
    </p></dd>

<dt>Using value*() functions</dt>
<dd><p>You can use the special functions
    <code>valueDouble</code>, <code>valueInt</code>, <code>valueLong</code>,
    <code>valueString</code> and <code>valueObject</code>
    to obtain the typed value of a column with a given name.
    The argument of the function is a string giving the exact
    (case-sensitive) column name, for instance
    <code>valueDouble("b_E(BP-RP)")</code>
    will yield the value of the column named "<code>b_E(BP-RP)</code>"
    as a double-precision floating point value.
    These functions are <em>not</em> the generally recommended way
    to get column values,
    since they are slower and provide less type-checking than
    the other options listed above,
    and can occasionally lead to some other esoteric problems.
    However, if you need to refer by name to strangely-named columns
    they are sometimes a convenient option.
    </p></dd>

<dt>With the Object$ prefix</dt>
<dd><p>If a column is referenced with the prefix "<code>Object$</code>"
    before its identifier
    (e.g. "<code>Object$BMAG</code>" for a column named <code>BMAG</code>)
    the result will be the column value as a java Object.
    Without that prefix, numeric columns are evaluated as java primitives.
    In most cases, you <em>don't want to do this</em>,
    since it means that you can't use the value in arithmetic expressions.
    However, if you need the value to be passed to a
    (possibly user-defined) method,
    and you need that method to be invoked even when the value is null,
    you have to do it like this.  Null-valued primitives
    otherwise cause expression evaluation to abort.
    </p></dd>

</dl>
</p>

<p>The value of the variables so referenced will be a primitive
(boolean, byte, short, char, int, long, float, double) if the 
column contains one of the corresponding types.  Otherwise it will
be an Object of the type held by the column, for instance a String.
In practice this means: you can write the name of a column, and it will
evaluate to the numeric (or string) value that that column contains
in each row.  You can then use this in normal algebraic expressions
such as "<code>B_MAG-U_MAG</code>" as you'd expect.
</p>

</subsect>

<subsect id="jel-paramref">
<subhead><title>Referencing Parameter Values</title></subhead>

<p>Some tables have constant values associated with them; these may
represent such things as the epoch at which observations were taken,
the name of the catalogue, an angular resolution associated with all 
observations, or any number of other things.  Such constants are known as
<em>table parameters</em> (not to be confused with parameters passed
to STILTS commands) and can be thought of as extra columns which have the
same value for every row.  The values of such parameters can be referenced
in STILTS algebraic expressions as follows:
<dl>
<dt>&param.prefix;<em>name</em></dt>
<dd><p>If the parameter name has a suitable form (starting with a letter
    and continuing with letters or numbers) it can be referenced by 
    prefixing that name with the string <code>&param.prefix;</code>.
    </p></dd>

<dt>&ucd.prefix;<em>ucd-spec</em></dt>
<dd><p>If the parameter has a
    <webref url="http://www.ivoa.net/Documents/latest/UCD.html"
            >Unified Content Descriptor</webref>
    it can be referenced by prefixing the UCD specifier with the
    string <code>&ucd.prefix;</code>.  Any punctuation marks in the
    UCD should be replaced by underscores, and a trailing underscore
    is interpreted as a wildcard.  See <ref id="jel-colref"/> for
    more discussion.
    </p></dd>

<dt>&utype.prefix;<em>utype-spec</em></dt>
<dd><p>If the parameter has a Utype,
    it can be referenced by prefixing the Utype specifier with the 
    string <code>&utype.prefix;</code>.  Any punctuation marks in the
    Utype should be replaced by underscores.  See <ref id="jel-colref"/> for
    more discussion.
    </p></dd>
</dl>
Note that if a parameter has a name in an unsuitable form (e.g. containing
spaces) and has no UCD then it cannot be referenced in an expression.
One possible workaround for that is to use the 
<ref id="fixcolnames"><code>fixcolnames</code></ref> filter.
</p>
</subsect>

<subsect id="jel-special">
<subhead><title>Special Tokens</title></subhead>

<p>There are a few pseudo-variables which have special functions
in the expression language.
The following specials are column-like, in that they have a different value
for each row:

<dl>
<dt>$index <em>or</em> $0</dt>
<dd><p>The value of this is the current row number (the first row is 1).
    Note that this value is a <code>long</code> (8-byte integer);
    when using it in certain expressions you may find it necessary to convert
    it to an <code>int</code> (4-byte integer) using the
    <code>toInteger()</code> function.
    <em>The deprecated alias "<code>INDEX</code>" may also be used.</em>
    </p></dd>

<dt>$random <em>(Deprecated)</em></dt>
<dd><p>Evaluates to a double-precision random number <code>0&lt;=x&lt;1</code>.
    <strong>NOTE:</strong> this token is deprecated since it can behave
    unpredictably (the same cell does not always yield the same result).
    Use instead the <code>random()</code> function in class
    <ref id="Maths" plaintextref="yes">Maths</ref>.
    </p></dd>
</dl>
</p>

<p>The following specials are parameter-like, in that their value is
not sensitive to the row:

<dl>
<dt>$ncol</dt>
<dd><p>The number of columns in the table.
    </p></dd>

<dt>$nrow</dt>
<dd><p>The number of rows in the table.  Note in some cases this is
    not known (e.g. if the table is being streamed), in which case
    the value of this variable is null.
    Note also that this value is a <code>long</code> (8-byte integer);
    when using it in certain expressions you may find it necessary to convert
    it to an <code>int</code> (4-byte integer) using the
    <code>toInteger()</code> function.
    </p></dd>
</dl>
</p>

</subsect>

<subsect>
<subhead><title>Null Values</title></subhead>

<p>When no special steps are taken, if a null value (blank cell)
is encountered
in evaluating an expression (usually because one of the columns
it relies on has a null value in the row in question) then the
result of the expression is also null.
</p>

<p>It is possible to exercise more control than this, but it
requires a little bit of care,
because the expressions work in terms of primitive values
(numeric or boolean ones) which don't in general have a defined null
value.  The name "<code>null</code>" 
in expressions gives you the java <code>null</code>
reference, but this cannot be matched against a primitive value
or used as the return value of a primitive expression.
</p>

<p>For most purposes, the following two tips should enable you to
work with null values:

<dl>
<dt>Testing for null</dt>
<dd><p>To test whether a column contains a null value, prepend the
    string "<code>NULL_</code>"
    (use upper case) to the column name or &column.id.char;ID.  This
    will yield a boolean value which is true if the column contains
    a blank or a floating point NaN (not-a-number) value,
    and false otherwise.
    Note that if combined with other boolean expressions,
    this null test should come first, i.e. write
    "<code>NULL_i || i==999</code>" rather than
    "<code>i==999 || NULL_i</code>",
    though this is only essential for integer or boolean variables.
    </p></dd>

<dt>Returning null</dt>
<dd><p>To return a null value from a numeric expression, use the name
    "<code>NULL</code>"
    (upper case).  To return a null value from a non-numeric expression
    (e.g. a String column) use the name "<code>null</code>" (lower case).
    </p></dd>
</dl>
</p>

<p>Null values are often used in conjunction with the conditional
operator, "<code>? :</code>"; the expression
<verbatim>
   test ? tval : fval
</verbatim>
returns the value <code>tval</code> if the boolean expression <code>test</code>
evaluates true, or <code>fval</code> if <code>test</code> evaluates false.
So for instance the following expression:
<verbatim>
   Vmag == -99 ? NULL : Vmag
</verbatim>
can be used to define a new column which has the same value as the 
<code>Vmag</code> column for most values, but if <code>Vmag</code> 
has the "magic" value -99 the new column will contain a blank.
The opposite trick (substituting a blank value with a magic one) can
be done like this:
<verbatim>
   NULL_Vmag ? -99 : Vmag
</verbatim>
Some more examples are given in <ref id="jelExamples"/>.
</p>

<p>Note that for floating point data,
STILTS treats <code>null</code> and NaN (Not-a-Number) values
somewhat interchangeably.
Blank values arising either from an input file format that can represent
missing values, or from processing that fails to provide a definite value,
are in most cases represented internally as <code>null</code> 
for integer-type values and NaN for floating point values.
However in general users should not rely on distinguishing between
<code>null</code> and NaN.
</p>

</subsect>

<subsect>
<subhead><title>Operators</title></subhead>

<p>The operators are pretty much the same as in the C language.
The common ones are:
<dl>
<dt>Arithmetic</dt>
<dd><p>
  <dl>
  <dt><code>+</code> (add)</dt>
  <dt><code>-</code> (subtract)</dt>
  <dt><code>*</code> (multiply)</dt>
  <dt><code>/</code> (divide)</dt>
  <dt><code>%</code> (modulus)</dt>
  </dl>
</p></dd>

<dt>Boolean</dt>
<dd><p>
  <dl>
  <dt><code>!</code> (not)</dt>
  <dt><code>&amp;&amp;</code> (and)</dt>
  <dt><code>||</code> (or)</dt>
  <dt><code>^</code> (exclusive-or)</dt>
  <dt><code>==</code> (numeric identity)</dt>
  <dt><code>!=</code> (numeric non-identity)</dt>
  <dt><code>&lt;</code> (less than)</dt>
  <dt><code>&gt;</code> (greater than)</dt>
  <dt><code>&lt;=</code> (less than or equal)</dt>
  <dt><code>&gt;=</code> (greater than or equal)</dt>
  </dl>
</p></dd>

<dt>Bitwise</dt>
<dd><p>
   <dl>
   <dt><code>&amp;</code> (and)</dt>
   <dt><code>|</code> (or)</dt>
   <dt><code>^</code> (exclusive-or)</dt>
   <dt><code>&lt;&lt;</code> (left shift)</dt>
   <dt><code>&gt;&gt;</code> (right shift)</dt>
   <dt><code>&gt;&gt;&gt;</code> (logical right shift)</dt>
   </dl>
</p></dd>

<dt>Numeric Typecasts</dt>
<dd><p>
  <dl>
  <dt><code>(byte)</code>   (numeric -> signed byte)</dt>
  <dt><code>(short)</code>  (numeric -> 2-byte integer)</dt>
  <dt><code>(int)</code>    (numeric -> 4-byte integer)</dt>
  <dt><code>(long)</code>   (numeric -> 8-byte integer)</dt>
  <dt><code>(float)</code>  (numeric -> 4-type floating point)</dt>
  <dt><code>(double)</code> (numeric -> 8-byte floating point)</dt>
  </dl>
  Note you may find the 
  <ref id="Maths" plaintextref="yes">Maths</ref> 
  conversion functions more convenient for numeric conversions than these.
</p></dd>

<dt>Other</dt>
<dd><p>
  <dl>
  <dt><code>+</code>  (string concatenation)</dt>
  <dt><code>[]</code> (array dereferencing - first element is zero)</dt>
  <dt><code>?:</code> (conditional switch)</dt>
  <dt><code>instanceof</code> (class membership)</dt>
  </dl>
</p></dd>
</dl>
</p>

</subsect>

<subsect id="quoting">
<subhead><title>Strings and Quoting</title></subhead>

<p>Sometimes in an algebraic expression you will want to
use a literal string value,
for instance if you want to test a string-valued
column for equality with some fixed string.
Literal strings must always be delimited by double quote characters
(<code>"</code>).
This can be problematic, since (single or double) quote characters
are interpreted within STILTS parameter values as grouping text
that may contain spaces into a single token.
And, depending how you are invoking STILTS,
the shell may do a similar thing:
interpret (single or double) quote characters
on the command line as grouping text that may contain spaces
or shell magic characters into a single string
(such as a parameter value assignment) to pass to STILTS.
To complicate matters further, literals of the <code>char</code> type
(single characters) in the expression language are delimited using
single-quote characters (<code>'</code>),
though it's not very often necessary to deal with
<code>char</code> values in STILTS.
And in a few cases specific parameters might have their own
requirements for quotes (like the <code>adql</code> parameter of
<ref id="tapquery"><code>tapquery</code></ref>;
in ADQL string literals need single quotes and
double quotes delimit identifiers).
</p>

<p>To summarise the common uses of quotes that you might have to make:
<ul>
<li>Delimit a literal string in the expression language:
    double quote (<code>"</code>)
    </li>
<li>Delimit a space-containing string (e.g. a list of tokens)
    that forms part of a STILTS parameter value assignment:
    single or double quote (<code>'</code> or <code>"</code>)
    </li>
<li>Protect a STILTS parameter value assignment
    from the Unix shell
    (avoid breaking into multiple words, ignore magic characters):
    single or double quote (<code>'</code> or <code>"</code>)
    </li>
</ul>
The last one only applies if you are running STILTS via
a Unix-like shell; similar considerations will apply from different OSs
like MS Windows,
but if you are running <ref id="jystilts">JyStilts</ref>
or using the expression language directly
e.g. from a GUI application like TOPCAT,
this one doesn't apply.
</p>

<p>Unfortunately, all this can lead to a kind of quoting hell
when trying to write STILTS commands,
especially if they have to contain String literals,
for which the author apologises :-(.
Working round these requirements can be quite messy,
but here are some tips:
<ul>
<li>String literals in the expression language must always be delimited with
    double quotes.
    </li>
<li>STILTS parameter values sometimes contain spaces or other characters
    that have meaning for the shell, so it's often a good idea to quote them;
    <code>ofmt=votable(format=BINARY)</code> will fail because the
    shell tries to interpret the parenthesis characters, but
    <code>ofmt='votable(format=BINARY)'</code>, or
    <code>'ofmt=votable(format=BINARY)'</code>, will succeed.
    </li>
<li>If you can avoid spaces in arguments, you won't need so much quoting,
    so you can write e.g. <code>cmd='select rmag&lt;12'</code>
    instead of <code>cmd='select "rmag &lt; 12"'</code>.
    </li>
<li>You can use different types of quote for different purposes,
    e.g. <code>cmd='keepcols "id ra dec"'</code>
    or   <code>cmd="keepcols 'id ra dec'"</code>.
    Note however that different types of quote have somewhat
    different meanings to the shell (single quotes are generally safer).
    </li>
<li>In a STILTS parameter value,
    as well in shell arguments,
    quotes can be escaped with a preceding backslash (<code>\</code>)
    to prevent them being interpreted as grouping constructs.
    So although <code>cmd='select equals(release,"DR4")'</code> fails
    (STILTS interprets the double quotes as grouping characters
    rather than string literal delimiters),
    <code>cmd='select equals(release,\"DR4\")'</code> succeeds.
    </li>
<li>In a STILTS parameter value,
    as well as in shell arguments,
    spaces can be escaped with a preceding backslash (<code>\</code>)
    to prevent them being interpreted as token separators.
    So <code>cmd='keepcols id\ ra\ dec'</code> can be used to mean the same
    as <code>cmd='keepcols "id ra dec"'</code>.
    </li>
</ul>
</p>

<p>Armed with this information it is usually possible to phrase
a STILTS command on the command line that does what you want.
In cases where that seems to be untrue or too painful,
there are a couple of ways to avoid use of the shell,
removing one layer of quote (mis)interpretation:
<ul>
<li>Filter (<ref id="tpipe-cmd"><code>cmd</code></ref>-like) parameters
    can be specified using file indirection;
    <code>cmd=@filename</code>
    reads the filter value(s) from the line(s) of the named file
    </li>
<li><ref id="jystilts">JyStilts</ref> allows you to use python syntax
    to construct arguments to pass to STILTS without interference
    from the shell
    </li>
</ul>
</p>

<p>The comments above concerning the Unix shell
are in principle dependent on
which shell is in use, but they should apply to
<code>sh</code>, <code>bash</code> and <code>csh</code>
on any common Unix-like OS including MacOS.
Something similar probably applies to other OSes like MS Windows,
but the details may be different.
</p>

</subsect>

<subsect id="staticMethods">
<subhead><title>Functions</title></subhead>

<p>Many functions are available for use within your expressions,
covering standard mathematical and trigonometric functions,
arithmetic utility functions, type conversions, and some more
specialised astronomical ones.
You can use them in just the way you'd expect,
by using the function name
(unlike column names, this is case-sensitive) followed by
comma-separated arguments in brackets, so
<verbatim>
    max(IMAG,JMAG)
</verbatim>
will give you the larger of the values in the columns IMAG and JMAG,
and so on.
</p>

<p>The functions available for use by default
are listed by class in the following subsections
with their arguments and short descriptions.
The <ref id="funcs"><code>funcs</code></ref> command provides another
way to browse these function descriptions online.
</p>

&jel.func.docs;

</subsect>

<subsect id="jelExamples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples for defining new columns;
the expressions below could appear as the <code>&lt;expr&gt;</code> in a
<code>tpipe</code> <code>addcol</code> or <code>sortexpr</code>
<ref id="filterSteps">command</ref>).
<dl>

<dt>Average</dt>
<dd><p><verbatim>
   (first + second) * 0.5
</verbatim></p></dd>

<dt>Square root</dt>
<dd><p><verbatim>
   sqrt(variance)
</verbatim></p></dd>

<dt>Angle conversion</dt>
<dd><p><verbatim>
   radiansToDegrees(DEC_radians)
   degreesToRadians(RA_degrees)
</verbatim></p></dd>

<dt>Conversion from string to number</dt>
<dd><p><verbatim>
   parseInt(&column.id.char;12)
   parseDouble(ident)
</verbatim></p></dd>

<dt>Conversion from number to string</dt>
<dd><p><verbatim>
   toString(index)
</verbatim></p></dd>

<dt>Conversion between numeric types</dt>
<dd><p><verbatim>
   toShort(obs_type)
   toDouble(range)
</verbatim><em>or</em><verbatim>
   (short) obs_type
   (double) range
</verbatim></p></dd>

<dt>Conversion from sexagesimal to degrees</dt>
<dd><p><verbatim>
   hmsToDegrees(RA1950)
   dmsToDegrees(decDeg,decMin,decSec)
</verbatim></p></dd>

<dt>Conversion from degrees to sexagesimal</dt>
<dd><p><verbatim>
   degreesToDms(&column.id.char;3)
   degreesToHms(RA,2)
</verbatim></p></dd>

<dt>Outlier clipping</dt>
<dd><p><verbatim>
   min(1000, max(value, 0))
</verbatim></p></dd>

<dt>Converting a magic value to null</dt>
<dd><p><verbatim>
   jmag == 9999 ? NULL : jmag
</verbatim></p></dd>

<dt>Converting a null value to a magic one</dt>
<dd><p><verbatim>
   NULL_jmag ? 9999 : jmag
</verbatim></p></dd>

<dt>Taking the third scalar element from an array-valued column</dt>
<dd><p><verbatim>
   psfCounts[2]
</verbatim></p></dd>

<dt>Converting spectral type to numeric value
    (e.g. "L3.5" -&gt; 23.5, "M7" -&gt; 17)</dt>
<dd><p><verbatim>
   "MLT".indexOf(spType.charAt(0)) * 10 + parseDouble(substring(spType,1)) + 10
</verbatim>
Note this uses a couple of Java
<webref url="&corejavadocs;/java/lang/String.html">String</webref>
instance methods (<ref id="instanceMethods"/>)
which are not explicitly documented in this section.
</p></dd>

</dl>

Here are some examples of boolean expressions that could be used
for row selection (appearing in a <code>tpipe</code> <code>select</code>
command)

<dl>
<dt>Within a numeric range</dt>
<dd><p><verbatim>
   <![CDATA[RA > 100 && RA < 120 && Dec > 75 && Dec < 85]]>
</verbatim></p></dd>

<dt>Within a circle</dt>
<dd><p><verbatim>
   &column.id.char;2*&column.id.char;2 + &column.id.char;3*&column.id.char;3 &lt; 1
   skyDistanceDegrees(ra0,dec0,hmsToDegrees(RA),dmsToDegrees(DEC))&lt;15./3600.
</verbatim></p></dd>

<dt>First 100 rows</dt>
<dd><p><verbatim>
   index &lt;= 100
</verbatim>
(though you could use <code>tpipe cmd='head 100'</code> instead)</p></dd>

<dt>Every tenth row</dt>
<dd><p><verbatim>
   index % 10 == 0
</verbatim>
(though you could use <code>tpipe cmd='every 10'</code> instead)</p></dd>

<dt>String equality/matching</dt>
<dd><p><verbatim>
   equals(SECTOR, "ZZ9 Plural Z Alpha")
   equalsIgnoreCase(SECTOR, "zz9 plural z alpha")
   startsWith(SECTOR, "ZZ")
   contains(ph_qual, "U")
</verbatim></p></dd>

<dt>String regular expression matching</dt>
<dd><p><verbatim>
   matches(SECTOR, "[XYZ] Alpha")
</verbatim></p></dd>

<dt>Test for non-blank value</dt>
<dd><p><verbatim>
   ! NULL_ellipticity
</verbatim></p></dd>
</dl>
</p>

</subsect>

<subsect id="jelAdvanced">
<subhead><title>Advanced Topics</title></subhead>

<p>This section contains some notes on getting the most out of
the algebraic expressions facility.  If you're not a Java programmer,
some of the following may be a bit daunting - read on at your
own risk!
</p>

<subsubsect>
<subhead><title>Expression evaluation</title></subhead>

<p>This note provides a bit more detail for Java programmers on what
is going on here; it describes how the use
of functions in STILTS algebraic expressions relates to normal Java
code.
</p>

<p>The expressions which you write are compiled to Java bytecode
when you enter them (if there is a 'compilation error' it will be
reported straight away).  The functions listed in the previous subsections
are all the <code>public static</code> methods of the classes which
are made available by default.  The classes listed are all in the
package <code>uk.ac.starlink.ttools.func</code>.
However, the public static methods are all imported into an anonymous
namespace for bytecode compilation, so that you write
(<code>sqrt(x,y)</code> and not <code>Maths.sqrt(x,y)</code>.
The same happens to other classes that are imported (which can be
in any package or none) - their public
static methods all go into the anonymous namespace.  Thus, method
name clashes are a possibility.
</p>

<p>This cleverness is all made possible by the rather wonderful
<webref url="http://www.gnu.org/software/jel/" plaintextref="yes">JEL</webref>.
</p>

</subsubsect>

<subsubsect id="instanceMethods">
<subhead><title>Instance Methods</title></subhead>

<p>There is another category of functions which can be used apart from
those listed in <ref id="staticMethods"/>.
These are called, in Java/object-oriented parlance, "instance methods"
and represent functions that can be executed on an object.
</p>

<p>It is possible to invoke any of its public
instance methods on any object
(though not on primitive values - numeric and boolean ones).
The syntax is that you place a "." followed by the method invocation
after the object you want to invoke the method on,
hence <code>NAME.substring(3)</code> instead of <code>substring(NAME,3)</code>.
If you know what you're doing, feel free to go ahead and do this.
However, most of the instance methods you're likely to want to use
have equivalents in the normal functions listed in the previous section,
so unless you're a Java programmer or feeling adventurous,
you may be best off ignoring this feature.
</p>

</subsubsect>


<subsubsect id="jelExtend">
<subhead><title>Adding User-Defined Functions</title></subhead>

<p>The functions provided by default for use with algebraic expressions,
while powerful, may not provide all the operations you need.
For this reason, it is possible to write your own extensions to the
expression language.  In this way you can specify abritrarily complicated
functions.
Note however that this will only allow you to define new columns or subsets
where each cell is a function only of the other cells in the same
row - it will not allow values in one row to be functions of values
in another.
</p>

<p>In order to do this, you have to write and compile a
(probably short) program
in the Java language.  A full discussion of how to go about this
is beyond the scope of this document, so if you are new
to Java and/or programming you may need to find a friendly local
programmer to assist (or mail the author).
The following explanation is aimed at Java programmers, but may not
be incomprehensible to non-specialists.
</p>

<p>The steps you need to follow are:
<ol>
<li>Write and compile a class containing one or more static public
    methods representing the function(s) required</li>
<li>Make this class available on the application's classpath at runtime
    as described in <ref id="jvmClasspath"/></li>
<li>Specify the class's name to the application, as the value of the
    <code>jel.classes</code>
    system property (colon-separated if there are several)
    as described in <ref id="sysProperties"/></li>
</ol>
</p>

<p>Any public static methods defined in the classes thus specified
will then be available for use.
They should be defined to take and return the relevant primitive or
Object types for the function required.
For instance a class written as follows would define a three-value average:
<verbatim>
   public class AuxFuncs {
       public static double average3(double x, double y, double z) {
           return (x + y + z) / 3.0;
       }
   }
</verbatim>
and the command
<verbatim>
   stilts tpipe cmd='addcol AVERAGE "average3(&column.id.char;1,&column.id.char;2,&column.id.char;3)"'
</verbatim>
would add a new column named AVERAGE giving the average of the first
three existing columns.
Exactly how you would build this is dependent on your system,
but it might involve doing something like the following:
<ol>
<li>Writing a file named <code>AuxFuncs.java</code> 
    containing the above code</li>
<li>Compiling it using a command like "<code>javac AuxFuncs.java</code>"</li>
<li>Running <code>tpipe</code> using the flags
    "<code>stilts -classpath . -Djel.classes=AuxFuncs tpipe</code>"</li>
</ol>
</p>

<p>Note that (in versions later than STILTS 3.0-6) variable-length argument
lists are supported where the final declared argument is an array,
so for instance a method declared like:
<verbatim>
       public static double sum(double[] values) {
           double total = 0;
           for (int i = 0; i &lt; values.length; i++) {
               total += values[i];
           }
           return total;
       }
</verbatim>
can be invoked like "<code>sum(UMAG,GMAG,RMAG,IMAG,ZMAG)</code>".
The alternative form "<code>double... values</code>" can be used in
the declaration with identical effect.
</p>

</subsubsect>

</subsect>

</sect>


<sect id="serverMode">
<subhead><title>Server Mode</title></subhead>

<p>STILTS is generally run by issuing commands from some kind
of command line.  However, facilities are also included to
run it as a service, accepting commands via HTTP.
The basic implementation of this is in the form of some
Servlet implementations that can be run in a servlet container.
At present the following servlets are provided:
<ul>
<li><javadoc docset="&stiltsjavadocs;"
             class="uk.ac.starlink.ttools.server.PlotServlet"/></li>
<li><javadoc docset="&stiltsjavadocs;"
             class="uk.ac.starlink.ttools.server.TaskServlet"/></li>
<li><javadoc docset="&stiltsjavadocs;"
             class="uk.ac.starlink.ttools.server.FormServlet"/></li>
</ul>
</p>

<p>In order to run them there are a few options:
<dl>

<dt>STILTS <code>server</code> command</dt>
<dd><p>The simplest is to use the STILTS
    <ref id="server"><code>server</code></ref> command.
    If you run this, the services will start up immediately and print
    information about the base URL.
    Some configuration of available functions and data access
    is offered by the command line parameters.
    This command is a thin wrapper around the Jetty servlet engine
    which is included in the STILTS distribution.
    </p></dd>

<dt>Deploy in a servlet container</dt>
<dd><p>For more control you can deploy the servlet or servlets
    you want to run in a servlet container of your choice,
    for instance Tomcat.  The servlets can be configured
    by setting context parameters in a <code>web.xml</code> file
    in the usual way; look at the 
    <javadoc docset="&stiltsjavadocs;"
             class="uk.ac.starlink.ttools.server.StiltsContext"/> class
    to see the available configuration parameters.
    </p></dd>

<dt>Docker</dt>
<dd><p>In the case of the plot server, a Docker container has been
    provided for convenience.
    At time of writing this is available, with some more discussion
    about how to deploy it, at
    <webref url="https://hub.docker.com/r/mbtaylor/plotserv"/>.
    </p></dd>

</dl>
In all cases, look at the base URL of the running service for more information
and usage examples.  For instance if the server is running at the
default base URL, then pointing your browser at
<verbatim>
    http://localhost:2112/stilts/
</verbatim>
will show you links to documentation for the available services.
</p>

<p>The different services are described further in the following subsections.
</p>

<p><strong>Note:</strong>
The <code>server</code> command and associated servlet code are
somewhat experimental.
If you have requirements which are not currently provided, please
contact the author for discussion.
</p>

<subsect id="PlotServlet">
<subhead><title>Plot Service</title></subhead>

<p>The plot server supports interactive plots that can be
viewed and navigated using HTTP requests.
A plot is specified with a URL, corresponding to an invocation
of one of the <ref id="plot2"><code>plot2*</code></ref> commands,
that specifies a plot, and uses
a service API that can be used by client-side javascript to
display and update a plot image in response to user navigation actions
such as mouse drag and wheel gestures.
The effect is of an interactive plot in a web browser that
can be navigated with the mouse in exactly the same way as
a STILTS or TOPCAT interactive plot.
</p>

<p>To see this in action, run
   <verbatim>
   % stilts server
   </verbatim>
point your browser at the URL displayed and follow the links
to see some interactive plots.
</p>

<p>The following subsections give some more details about how
this works and how it can be used.
</p>

<subsubsect id="plotserv-use">
<subhead><title>Usage</title></subhead>

<p>The best way to see how the plot service can be used is to run the service
(most simply by executing <code>stilts <ref id="server">server</ref></code>),
and look at the running plot examples, along with their HTML source code,
served from the server endpoint.
</p>

<p>However this section gives a brief independent overview
of how the service can be used.
The PlotServlet itself exposes a RESTful API as described in
<ref id="plotserv-api"/>.
This offers the services required for client HTML/JavaScript code
running in a browser to set up a plot based on server-side
(or at least server-accessible) table data, and to make updates to it
following user activity like mouse gestures.
</p>

<p>A basic client JavaScript library named <code>plot2Lib.js</code>,
that facilitates this plot setup is included with STILTS;
it can be found in the <code>stilts.jar</code> file at
<code>uk/ac/starlink/ttools/server/plot2Lib.js</code>,
and is available from the running server at
<code>&lt;plot-service-base&gt;/plot2Lib.js</code>.
It exports some variables in the <code>plot2</code> namespace;
see comments in the file itself for documentation of how to use it.
A minimal web page using this to present an interactive plot might look like:
<verbatim><![CDATA[
   <html>
   <script src="plot2Lib.js"></script>
   <script>
      var serverUrl = "http://localhost:2112/stilts/plot/";
      onload = function() {
         var plotTxt = plot2.wordsToPlotTxt([
            "plot2sky",
            "layer1=mark",
            "in1=/data/catalogue.fits",
            "lon1=RA",
            "lat1=DEC"
         ]);
         var plotNode = plot2.createPlotNode(serverUrl, plotTxt);
         document.getElementsByTagName("body")[0].appendChild(plotNode);
      };
   </script>
   </html>
]]></verbatim>
In this case the data file "<code>/data/catalogue.fits</code>"
has to be available to the servlet;
how this is arranged depends on the way the service is deployed.
</p>

<p>JavaScript clients don't have to use the <code>plot2Lib.js</code> library;
it's also possible to write custom code that talks to the
RESTful API described in <ref id="plotserv-api"/>.
</p>

</subsubsect>

<subsubsect id="plotserv-api">
<subhead><title>RESTful API</title></subhead>

<p>This section details the endpoints supported by the plot servlet;
it is not required reading for those who want to use the supplied
<code>plot2Lib.js</code> JavaScript library,
but may be of interest to those who want to write their own
JavaScript clients.
</p>

&plotserv-syntax;

<p>This API is subject to extension or modification in future releases.
</p>

</subsubsect>

<subsubsect id="plotserv-cache">
<subhead><title>Caching and Performance</title></subhead>

<p>The plot servlet requires some temporary disk storage for caching
images and prepared coordinate data to improve performance when
updating plots during visualisation sequences.
Servlet configuration options are available to manage usage
of these resources.
</p>

<p>When first requested to make a plot, it works out what coordinate
data will be required, reads this from the input table,
and writes it to cache in an efficient binary storage format.
This may be simply a copy of columns from the input table,
or may require some computation, for instance if the
plotted quantity is the result of an expression in the expression
language, or if the coordinates are longitude and latitude
(in which case they are converted to unit vector components).
This coordinate cache preparation requires a scan of the input
table which is in the current implementation single-threaded,
so may be time-consuming for very large tables.
</p>

<p>Once the coordinate data is in cache, the cache rather than
the input table is used for subsequent plots; that includes both
derived replots resulting from user navigation and completely separate
plots that happen to require the same row/column data.
Plots from cache are in most cases multi-threaded, so can
be quite rapid even for large datasets depending on server
configuration and load.
</p>

<p>When the plot is first made, the initial (pre-navigation) plotted image is
also stored in a cache, so that if other HTTP clients request
the same plot they can receive the initial image without further
data access or computation.
</p>

<p>By default, the directory used for caching is the value of the
<code>java.io.tmpdir</code> system property, which is typically
<code>/tmp</code> on Unix systems.
The directory used for caching can be changed by modifying this
system property, or caching can be configured with
the servlet context parameters controlled by the
<javadoc docset="&stiltsjavadocs;"
         class="uk.ac.starlink.ttools.server.StiltsContext"/> class.
</p>

<p>When cache storage gets full, older items are dropped in the usual way.
There is no doubt scope for improvement of the existing cache management;
the details may be refined in future releases.
</p>

</subsubsect>

</subsect>

<subsect id="TaskServlet">
<subhead><title>Task Service</title></subhead>

<p>This servlet allows execution of a single STILTS command
by making an HTTP request with the command name and parameters,
rather than by presenting them on the command line.
You might want to run this service if you are providing
a web service to external users which is able to access files residing
on the server, for instance generating static table plots
(though see the <ref id="PlotServlet">Plot service</ref> for
more sophisticated server-side visualisation)
or row selections on the fly.
This can be done without the server mode, for instance by invoking
the <code>stilts</code> script or java from a CGI script to serve
each request, but using server mode has two advantages:
first it provides correct output HTTP headers such as Content-Type,
and secondly it avoids the Java startup overheads for each invocation.
</p>

<p>The URLs accepted by the service are of the form
<verbatim><![CDATA[
   <task-base-url>/<task-name>?<arg-name>=<arg-value>&<arg-name>=<arg-value>...
]]></verbatim>
The <code>&lt;task-name&gt;</code> is one of the STILTS command names
as listed in <ref id="cmdUsage"/>,
and the <code>&lt;arg-name&gt;=&lt;arg-value&gt;</code> pairs
are the parameter settings for that command;
the parameter settings are syntactically
an <code>application/x-www-form-urlencoded</code> query string.
These parameters may be supplied as a POST body rather than as part of a
GET query if preferred, though since the task invocation will normally
be idempotent, GET may be more respectable.
</p>

<p>The HTTP response depends on the behaviour of the task in question.
If the task is one that outputs a table or sequence of tables, such as
<ref id="tpipe"><code>tpipe</code></ref> or
<ref id="tmatch2"><code>tmatch2</code></ref>,
the HTTP response will be the serialized form of the table
in an appropriate format.  By default that is currently VOTable
(Content-Type "<code>application/x-votable+xml</code>"),
but that can be altered by supplying the appropriate output
format parameter, e.g. "<code>ofmt=fits</code>".
Plotting commands return an image in a default graphics format;
again the output format can be controlled using command paramters
in the same way as on the command line.
Output for some other commands such as <ref id="calc"><code>calc</code></ref>
may just be plain text.
</p>

<p>Here are some example invocations, assuming the default
server base URL of <code>http://localhost:2112/stilts/</code>:
<dl>
    
<dt><verbatim>
http://localhost:2112/stilts/
</verbatim></dt>
<dd><p>Returns an HTML page giving version information and some links to
    example usages of the server.
    </p></dd>

<dt><verbatim>
http://localhost:2112/stilts/task/tpipe
</verbatim></dt>
<dd><p>Returns an HTML page giving usage instructions for the
    <ref id="tpipe"><code>tpipe</code></ref> task.
    </p></dd>

<dt><verbatim>
http://localhost:2112/stilts/task/calc?expression=21%2b2
</verbatim></dt>
<dd><p>Invokes the <ref id="calc"><code>calc</code></ref> task
    to return a document containing the text "<code>23</code>".
    Note that the plus ("<code>+</code>") sign in the expression has to be
    encoded using the sequence "<code>%2b</code>" since "<code>+</code>"
    has a special significance in query URLs - see for instance sec 2.2 of
    <webref url="http://www.ietf.org/rfc/rfc1738.txt">RFC 1738</webref>.
    </p></dd>

<dt><verbatim>
http://localhost:2112/stilts/task/plot2plane?in=/data/table1.vot&amp;layer1=mark&amp;x1=RMAG&amp;y1=BMAG
</verbatim></dt>
<dd><p>Invokes the <ref id="plot2d"><code>plot2d</code></ref> task 
    to return a magnitude-magnitude diagram of the named server-side file
    as an image, probably an <code>image/png</code>
    (but see the <ref id="PlotServlet">Plot Service</ref>
    if you want interactive plots).
    </p></dd>

<dt><verbatim>
http://localhost:2112/stilts/task/tcopy?in=/data/cat.fits&amp;ofmt=ecsv
</verbatim></dt>
<dd><p>Invokes the <ref id="tcopy"><code>tcopy</code></ref> task
    to return a translation of the named server-side FITS file to ECSV format.
    </p></dd>

</dl>
</p>

<p>Note that in the current form of this service
no great attention has been paid
to security, so it may be possible for clients to read and write files
and expend significant system resources by making certain requests to the
server.  Anyone exposing this service directly to external
clients should bear this in mind.
</p>

</subsect>

<subsect id="FormServlet">
<subhead><title>Form Service</title></subhead>

<p>The Form service is more or less obsolete;
it really just gives an example of how to set up an HTML Form
to invoke one of the STILTS commands using the
<ref id="TaskServlet">Task Service</ref>.
</p>

<p>The content served gives provides a couple of examples of an HTML form
that can invoke plotting using the <ref id="TaskServlet">Task Service</ref>.
The output graphics use the <ref id="plot">old-style plot commands</ref>
rather than the much more capable <ref id="plot2"><code>plot2*</code></ref>
plotting, and are not interactive;
for much improved server-side visualisastion,
see <ref id="PlotServlet"/>.
</p>

</subsect>

</sect>


<sect id="taskApi">
<subhead><title>Programmatic Invocation</title></subhead>

<p>The STILTS package provides some capabilities,
for instance plotting, that might be useful
as part of other Java applications.
The code that forms STILTS is fully documented at the API level;
there are comprehensive javadocs throughout for the
<code>uk.ac.starlink.ttools</code> package, its subpackages,
and most of the other classes in the <code>uk.ac.starlink</code>
tree on which it relies.
Anybody is welcome to use these classes at their own risk,
but the code does not form a stable API intended for public use:
the javadocs are not distributed as part of the package
(though you may be able to find them
<webref url="&stiltsjavadocs;">here</webref>),
tutorial documentation is not provided, and there is no commitment
to API stability between releases.
</p>

<p>With this in mind, there are facilities for invoking the
STILTS commands programmatically from third-party java code.
Of course it is possible to do this by just calling the
static <code>main(String[])</code> method of the application
Main-Class
(<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.ttools.Stilts"/>)
but we document here how it can be done in a way which
allows more control, using the
<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.task."/>
parameter handling framework.
</p>

<p>Each of the STILTS tasks listed in <ref id="cmdUsage"/>
is represented by a class implementing the 
<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.task.Task"/> interface;
these all have no-arg constructors.
To run it, you need to create an instance of the class,
pass it an
<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.task.Environment"/>
object which can acquire values for parameters by name, and then execute it.
The
<javadoc docset="&stiltsjavadocs;"
         class="uk.ac.starlink.ttools.task.MapEnvironment"/>
class, based on a Map containing name/value pairs,
is provided for this purpose.
As well as managing parameter values, MapEnvironment captures
table and text output in a way that lets you retrieve it after
the task has executed.
Here is a simple example for invoking the <ref id="calc">calc</ref> task
to perform a simple calcation:
<verbatim>
    MapEnvironment env = new MapEnvironment();
    env.setValue( "expression", "sqrt(3*3+4*4)" );
    Task calcTask = new uk.ac.starlink.ttools.task.Calc();
    calcTask.createExecutable( env ).execute();
    String result = env.getOutputText();
</verbatim>
The execution corresponds exactly to the command-line:
<verbatim>
    stilts calc expression="sqrt(3*3+4*4)"
</verbatim>
The <ref id="calc-usage">Usage</ref> section for the <code>calc</code>
task notes that the corresponding Task subclass is
<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.ttools.task.Calc"/>.
</p>

<p>Also in the usage section, each parameter reports the data type that
it may take, and objects of this type may be used as the parameter
value passed in the <code>MapEnvironment</code> as an alternative
to passing string values.
For the case of the input table parameters,
this is
<javadoc docset="&stiltsjavadocs;" class="uk.ac.starlink.table.StarTable"/>,
so in a task like
<ref id="tpipe">tpipe</ref>
(<javadoc docset="&stiltsjavadocs;"
          class="uk.ac.starlink.ttools.task.TablePipe"/>),
if you want to read a file "data.fits",
you can either write
<verbatim>
    env.setValue( "in", "data.fits" );
</verbatim>
or
<verbatim>
    StarTable table = new StarTableFactory().readStarTable( "data.fits" );
    env.setValue( "in", table );
</verbatim>
That doesn't buy you much, but the table could equally be obtained
from any other source, including being a user-defined iterable
over existing data structures.
See <docxref doc="sun252"/> for more information on
<code>StarTable</code> handling.
</p>

<p>For some short examples of programs which invoke STILTS tasks
in this way, see the source code of some of the examples in the
<code>uk.ac.starlink.ttools.example</code> directory:
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/Calculator.java">Calculator</webref> and
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/Head10.java">Head10</webref>.
</p>

<p>Some commands provide additional methods for use with parameter-based
invocation.  In particular the plotting commands can be used to
create JComponent objects that can be incorporated into an
existing GUI.
A working example of this
can be found in the source code for the example
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/EnvPlanePlotter.java">EnvPlanePlotter</webref>
class.
For some more tutorial introductions to using the plotting classes
programmatically, see also the example classes
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/SinePlot.java">SinePlot</webref>,
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/ApiPlanePlotter.java">ApiPlanePlotter</webref>,
and
<webref url="&starjavasrc;ttools/src/main/uk/ac/starlink/ttools/example/BasicPlotGui.java">BasicPlotGui</webref>
in the same place.
</p>

</sect>

<appendices>

<sect id="classified">
<subhead><title>Commands By Category</title></subhead>

<p>This section lists the commands available broken down by the
category of function they provide.  Some commands appear in more than
one category.
Detailed descriptions and examples for each command can be found in
<ref id="cmdUsage"/>.
<dl>

<dt>Format conversion:</dt>
<dd><p>
    <ul>
    <li><ref id="tcopy" plaintextref="yes"><code>tcopy</code></ref>:
        &tcopy-purpose;
        </li>
    <li><ref id="votcopy" plaintextref="yes"><code>votcopy</code></ref>:
        &votcopy-purpose;
        </li>
    </ul>
    See also <ref id="io" plaintextref="yes"/>.
    </p></dd>

<dt>Generic table manipulation:</dt>
<dd><p>
    <ul>
    <li><ref id="tcopy" plaintextref="yes"><code>tcopy</code></ref>:
        &tcopy-purpose;
        </li>
    <li><ref id="tpipe" plaintextref="yes"><code>tpipe</code></ref>:
        &tpipe-purpose;
        </li>
    <li><ref id="tmulti" plaintextref="yes"><code>tmulti</code></ref>:
        &tmulti-purpose;
        </li>
    <li><ref id="tmultin" plaintextref="yes"><code>tmultin</code></ref>:
        &tmultin-purpose;
        </li>
    <li><ref id="tcat" plaintextref="yes"><code>tcat</code></ref>:
        &tcat-purpose;
        </li>
    <li><ref id="tcatn" plaintextref="yes"><code>tcatn</code></ref>:
        &tcatn-purpose;
        </li>
    <li><ref id="tloop" plaintextref="yes"><code>tloop</code></ref>:
        &tloop-purpose;
        </li>
    <li><ref id="tjoin" plaintextref="yes"><code>tjoin</code></ref>:
        &tjoin-purpose;
        </li>
    <li><ref id="arrayjoin" plaintextref="yes"><code>arrayjoin</code></ref>:
        &arrayjoin-purpose;
        </li>
    <li><ref id="tgridmap" plaintextref="yes"><code>tgridmap</code></ref>:
        &tgridmap-purpose;
        </li>
    <li><ref id="tgroup" plaintextref="yes"><code>tgroup</code></ref>:
        &tgroup-purpose;
        </li>
    <li><ref id="tcube" plaintextref="yes"><code>tcube</code></ref>:
        &tcube-purpose;
        </li>
    </ul>
    See also <ref id="pipes" plaintextref="yes"/>.
    </p></dd>

<dt>Crossmatching:</dt>
<dd><p>
    <ul>
    <li><ref id="tmatch1" plaintextref="yes"><code>tmatch1</code></ref>:
        &tmatch1-purpose;
        </li>
    <li><ref id="tmatch2" plaintextref="yes"><code>tmatch2</code></ref>:
        &tmatch2-purpose;
        </li>
    <li><ref id="tmatchn" plaintextref="yes"><code>tmatchn</code></ref>:
        &tmatchn-purpose;
        </li>
    <li><ref id="tskymatch2" plaintextref="yes"><code>tskymatch2</code></ref>:
        &tskymatch2-purpose;
        </li>
    <li><ref id="cdsskymatch" plaintextref="yes"><code>cdsskymatch</code></ref>:
        &cdsskymatch-purpose;
        </li>
    <li><ref id="coneskymatch"
             plaintextref="yes"><code>coneskymatch</code></ref>:
        &coneskymatch-purpose;
        </li>
    <li><ref id="sqlskymatch" plaintextref="yes"><code>sqlskymatch</code></ref>:
        &sqlskymatch-purpose;
        </li>
    </ul>
    See also <ref id="match" plaintextref="yes"/>.
    </p></dd>

<dt>Plotting:</dt>
<dd><p>
    <ul>
    <li><ref id="plot2plane" plaintextref="yes"><code>plot2plane</code></ref>:
        &plot2plane-purpose;
        </li>
    <li><ref id="plot2sky" plaintextref="yes"><code>plot2sky</code></ref>:
        &plot2sky-purpose;
        </li>
    <li><ref id="plot2cube" plaintextref="yes"><code>plot2cube</code></ref>:
        &plot2cube-purpose;
        </li>
    <li><ref id="plot2sphere" plaintextref="yes"><code>plot2sphere</code></ref>:
        &plot2sphere-purpose;
        </li>
    <li><ref id="plot2corner" plaintextref="yes"><code>plot2corner</code></ref>:
        &plot2corner-purpose;
        </li>
    <li><ref id="plot2time" plaintextref="yes"><code>plot2time</code></ref>:
        &plot2time-purpose;
        </li>
    <li><ref id="plot2d" plaintextref="yes"><code>plot2d</code></ref>
        <em>(deprecated)</em>:
        &plot2d-purpose;
        </li>
    <li><ref id="plot3d" plaintextref="yes"><code>plot3d</code></ref>
        <em>(deprecated)</em>:
        &plot3d-purpose;
        </li>
    <li><ref id="plothist" plaintextref="yes"><code>plothist</code></ref>
        <em>(deprecated)</em>:
        &plothist-purpose;
        </li>
    </ul>
    See also <ref id="plot" plaintextref="yes"/>.
    </p></dd>

<dt>Sky Pixel Operations:</dt>
<dd><p>
    <ul>
    <li><ref id="tskymap" plaintextref="yes"><code>tskymap</code></ref>:
        &tskymap-purpose;
        </li>
    <li><ref id="pixfoot" plaintextref="yes"><code>pixfoot</code></ref>:
        &pixfoot-purpose;
        </li>
    <li><ref id="pixsample" plaintextref="yes"><code>pixsample</code></ref>:
        &pixsample-purpose;
        </li>
    </ul>
    </p></dd>

<dt>VOTables:</dt>
<dd><p>
    <ul>
    <li><ref id="votcopy" plaintextref="yes"><code>votcopy</code></ref>:
        &votcopy-purpose;
        </li>
    <li><ref id="votlint" plaintextref="yes"><code>votlint</code></ref>:
        &votlint-purpose;
        </li>
    </ul>
    </p></dd>

<dt>Virtual Observatory service access:</dt>
<dd><p>
    <ul>
    <li><ref id="cdsskymatch" plaintextref="yes"><code>cdsskymatch</code></ref>:
        &cdsskymatch-purpose;
        </li>
    <li><ref id="cone" plaintextref="yes"><code>cone</code></ref>:
        &cone-purpose;
        </li>
    <li><ref id="coneskymatch"
             plaintextref="yes"><code>coneskymatch</code></ref>:
        &coneskymatch-purpose;
        </li>
    <li><ref id="tapskymatch" plaintextref="yes"><code>tapskymatch</code></ref>:
        &tapskymatch-purpose;
        </li>
    <li><ref id="tapquery" plaintextref="yes"><code>tapquery</code></ref>:
        &tapquery-purpose;
        </li>
    <li><ref id="tapresume" plaintextref="yes"><code>tapresume</code></ref>:
        &tapresume-purpose;
        </li>
    <li><ref id="taplint" plaintextref="yes"><code>taplint</code></ref>:
        &taplint-purpose;
        </li>
    <li><ref id="datalinklint"
             plaintextref="yes"><code>datalinklint</code></ref>:
        &datalinklint-purpose;
        </li>
    <li><ref id="regquery" plaintextref="yes"><code>regquery</code></ref>:
        &regquery-purpose;
        </li>
    </ul>
    </p></dd>

<dt>SQL Database access:</dt>
<dd><p>
    <ul>
    <li><ref id="sqlclient" plaintextref="yes"><code>sqlclient</code></ref>:
        &sqlclient-purpose;
        </li>
    <li><ref id="sqlupdate" plaintextref="yes"><code>sqlupdate</code></ref>:
        &sqlupdate-purpose;
        </li>
    <li><ref id="sqlskymatch" plaintextref="yes"><code>sqlskymatch</code></ref>:
        &sqlskymatch-purpose;
        </li>
    </ul>
    </p></dd>

<dt>Miscellaneous:</dt>
<dd><p>
    <ul>
    <li><ref id="server" plaintextref="yes"><code>server</code></ref>:
        &server-purpose;
        </li>
    <li><ref id="calc" plaintextref="yes"><code>calc</code></ref>:
        &calc-purpose;
        </li>
    <li><ref id="funcs" plaintextref="yes"><code>funcs</code></ref>:
        &funcs-purpose;
        </li>
    <li><ref id="xsdvalidate" plaintextref="yes"><code>xsdvalidate</code></ref>:
        &xsdvalidate-purpose;
        </li>
    </ul>
    </p></dd>

</dl>
</p>

</sect>

<sect id="cmdUsage">
<subhead><title>Command Reference</title></subhead>

<p>This appendix provides the reference documentation for the
commands in the package.  For each one a description of its purpose,
a list of its command-line arguments, and some examples are given.
</p>

<subsect id="arrayjoin" tocleaf="yes">
<subhead><title><code>arrayjoin</code>: &arrayjoin-purpose;</title></subhead>

<p><code>arrayjoin</code> takes an input table and for each row
adds the contents of a separate "array" table.
The columns added are the columns from the array table,
and the value of each cell is the value of the whole column from
the array table represented as an array.
The assumption is that all the array tables have the same form
(the same columns, though not necessarily the same row counts).
</p>

<p>This can be useful for constructing a single table
with array-valued columns containing data that is made
available in multiple external files, for instance
via the DataLink protocol;
this is illustrated in the Examples subsection below.
Note however that this command does not understand DataLink directly,
and cannot itself determine the location of the external array tables;
an expression giving their per-row location (filename or URL)
must be supplied.
</p>

&arrayjoin-summary;

<subsubsect id="arrayjoin-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>arrayjoin</code>:
<dl>

<dt><verbatim>
stilts arrayjoin in=dr3-sources.vot
                 atable='"https://gea.esac.esa.int/data-server/data?RETRIEVAL_TYPE=XP_SAMPLED&amp;RELEASE=Gaia+DR3&amp;ID="+source_id'
                 icmd=progress
                 out=sources-with-xp.fits
</verbatim></dt>
<dd><p>This command uses the Gaia DR3 DataLink service
    to attach sampled XP spectrum data to an input table containing
    other information about Gaia DR3 sources.
    </p>
    <p>The input file <code>dr3-sources.vot</code> must contain the column
    <code>source_id</code> giving the Gaia DR3 source ID
    corresponding to that row.
    The <code>atable</code> expression is a URL built using that
    <code>source_id</code> column and a base URL that can be obtained
    by examining the DR3 catalogue
    <webref url='https://www.cosmos.esa.int/web/gaia-users/archive/datalink-products#datalink_serialisation'
            >documentation</webref>,
    or by examining the DataLink service descriptor returned from a
    source catalogue query.
    </p>
    <p>The output file <code>sources-with-xp.fits</code>
    is a FITS table with the same content as the input
    but with three added array-valued columns,
    <code>wavelength</code>, <code>flux</code> and <code>flux_error</code>
    as supplied by tables downloaded from the URL in the <code>atable</code>
    parameter.
    For each row of the output table, the lengths of those three arrays
    will be the same, namely the row count of the table that supplied them.
    These matched arrays can be manipulated within STILTS,
    for instance using the
    <ref id="Arrays">Arrays</ref> functions
    or to produce plots using the
    <ref id="layer-lines">lines</ref> or
    <ref id="layer-arrayquantile">arrayquantile</ref> plot layer types.
    </p>
    <p>The <code>icmd=progress</code> filter is a useful convenience to see
    how the command is progressing,
    since the operation requires multiple downloads and so may be
    time-consuming.
    </p>
    </dd>

<dt><verbatim>
stilts arrayjoin in=dr3-sources.vot
                 atable='"https://gea.esac.esa.int/data-server/data?RETRIEVAL_TYPE=XP_SAMPLED&amp;RELEASE=Gaia+DR3&amp;ID="+source_id'
                 icmd=progress
                 keepall=false
                 ocmd='constcol -acceptnull'
                 out=sources-with-xp.fits
</verbatim></dt>
<dd><p>Similar to the previous example, but with a couple of variations.
    </p>
    <p>The <code>keepall=false</code> parameter means that only those
    rows for which array data exists will be retained in the output table,
    rather than including blank entries in the array columns in cases
    for which the remote table couldn't be loaded.
    </p>
    <p>The <code>ocmd='constcol -acceptnull'</code> filter looks for
    columns in the output that have the same value (or null) in every row.
    Since Gaia DR3 sampled spectra are all sampled onto the same grid,
    in this case the <code>wavelength</code> column will have the same
    array value for every row, which is harmless but makes the output table
    larger than required.
    The <ref id="constcol"><code>constcol</code></ref> filter
    spots this and removes the <code>wavelength</code> column,
    replacing it with a parameter having the same name,
    which can therefore be referred to using the
    <ref id="jel-paramref">expression language</ref>
    as <code>&param.prefix;wavelength</code>.
    </p></dd>

<dt><verbatim>
stilts arrayjoin in=dl_dr3.fits
                 icmd='select has_epoch_photometry'
                 atable='"https://gea.esac.esa.int/data-server/data?RETRIEVAL_TYPE=EPOCH_PHOTOMETRY&amp;ID="+urlEncode(designation)'
                 afmt=votable
                 acmd='select equals(BAND,\"G\")'
                 acmd='keepcols "time mag flux"'
                 out=with-epoch-photom.fits
</verbatim></dt>
<dd><p>Similar to the first example, but this time acquiring
    EPOCH_PHOTOMETRY rather than XP_SAMPLED data,
    and performing some additional
    processing of the main and array input tables.
    The value of the <code>designation</code> field is wrapped in the
    <ref id="URLs-urlEncode"><code>urlEncode</code></ref>
    function when preparing the URL;
    this is necessary since that field may contain spaces which
    are not legal URL characters.
    Only those input rows are used for which the boolean
    <code>has_epoch_photometry</code> column is True;
    for the linked array tables the rows are limited to those with
    the <code>BAND</code> column having the value "<code>G</code>";
    and only the array columns
    <code>time</code>, <code>mag</code> and <code>flux</code>
    are retained for inclusion into the output.
    </p></dd>

<!--
 ! stilts tapquery sync=true
 !        tapurl=https://gaia.ari.uni-heidelberg.de/tap adql="select top 10 source_id, designation, ra, dec, pf, pf_error from gaiadr3.gaia_source join gaiadr3.vari_rrlyrae using (source_id) where has_epoch_photometry='true'"
 !        out=rrlyrae.vot
 !-->
<dt><verbatim>
stilts arrayjoin in=rrlyrae.vot
                 atable='"https://gaia.ari.uni-heidelberg.de/timeseries/gaiadr3?sourceid="+source_id'
                 acmd='keepcols "time mag flux flux_error"'
                 aparams='p1 pf'
                 icmd=progress
                 out=rrlyrae-timeseries.vot
</verbatim></dt>
<dd><p>Acquires Gaia DR3 epoch photometry from a different service, ARI-Gaia.
    In this case the epoch photometry tables have parameters
    characterising the variability;
    the <code>aparams='p1 pf'</code> parameter takes two of these and
    adds them as scalar-valued columns in the output table.
    Since the <code>pf</code> parameter gives the variability period,
    a period-folded variability graph can then be plotted using
    an X coordinate like <code>arrayFunc("x%1",divide(time,pf))</code>.
    </p></dd>

<dt><verbatim>
stilts arrayjoin in=obscore-lotssdr2.vot
                 atable=access_url
                 icmd=progress
                 icmd='select equals(dataproduct_type,\"spectrum\")'
                 out=with-spectra.vot
</verbatim></dt>
<dd><p>Takes an <webref url="https://www.ivoa.net/documents/ObsCore/"
                        >ivoa.ObsCore</webref>-like table
       and adds array-valued columns giving the content of spectrum files
       linked from the <code>access_url</code> field.
       Only those rows with a <code>dataproduct_type</code>
       of "<code>spectrum</code>" are included.
       Note all the access_urls in the selected rows must point to
       spectrum files of a similar form (i.e. having the same columns),
       otherwise the command will fail.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>


<subsect id="calc" tocleaf="yes">
<subhead><title><code>calc</code>: &calc-purpose;</title></subhead>

<p><code>calc</code> is a very simple utility for evaluating expressions.
It uses the same expression evaluator as is used in <code>tpipe</code>
and the other generic table tasks for things like creating new columns,
so it can be used as a quick test to see what expressions work,
or in order to evaluate expressions using the various algebraic
functions documented in <ref id="staticMethods"/>.
Since usually no table is involved, you can't refer to column names in
the expressions.
It has one mandatory parameter, the expression to evaluate, and writes the
result to the screen.
</p>

&calc-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>calc</code>:
<dl>

<dt><verbatim>
stilts calc 1+2
</verbatim></dt>
<dd><p>Calculates one plus two.  Writes "3" to standard output.
    </p></dd>

<dt><verbatim>
stilts calc 'isoToMjd("2005-12-25T00:00:00")'
</verbatim></dt>
<dd><p>Works out the Modified Julian Day corresponding to Christmas 2005.
    The output is "53729.0".
    </p></dd>

<dt><verbatim>
stilts calc '&param.prefix;author' table=catalogue.xml
</verbatim></dt>
<dd><p>In this case the expression is evaluated in the context of the
    supplied table, which means that the table's parameters can be
    referenced in the expression.  This example just outputs the
    value of the table parameter named "author".
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="cdsskymatch" tocleaf="yes">
<subhead><title><code>cdsskymatch</code>:
                &cdsskymatch-purpose;</title></subhead>

<p><code>cdsskymatch</code> uses the
<webref url="http://cdsxmatch.u-strasbg.fr/">CDS X-Match service</webref>
to join a local table to one of the tables hosted by the
Centre de Donn&#x00e9;es astronomiques de Strasbourg.
This includes all of the VizieR tables and the SIMBAD database.
The service is very fast, and in most cases it is the best way to
match a local table against a large external table hosted by a service.
It is almost certainly much better than using
<ref id="coneskymatch"><code>coneskymatch</code></ref>,
though it is less flexible than TAP
(see the <ref id="tapquery"><code>tapquery</code></ref> task
for flexible access to TAP services, and
<ref id="tapskymatch"><code>tapskymatch</code></ref>
for positional matches).
</p>

<p>The local table is uploaded to the X-Match service in chunks,
and the matches for each chunk are retrieved in turn and eventually
stitched together to form the final result.
The tool only uploads sky position and an identifier for each row of
the input table, but all columns of the input table are reinstated
in the result for reference.
</p>

<p>For a better understanding of the details of how this service operates,
including exactly what coordinates are matched against the uploaded positions
(roughly: integrated to J2000 using proper motions if available)
and what columns are included in the output
(roughly: a subset of the most commonly used columns),
please consult the
<webref url="http://cdsxmatch.u-strasbg.fr/xmatch/doc/"
        >service documentation</webref>.
</p>

<p><em>Acknowledgement</em>: CDS note that if the use of the
X-Match service is useful to your research, they would appreciate
the following acknowledgement:
<blockquote>
<em>
  "This research made use of the cross-match service provided by
  CDS, Strasbourg."
</em>
</blockquote>
</p>

&cdsskymatch-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>cdsskymatch</code>:
<dl>

<dt><verbatim>
stilts cdsskymatch cdstable=II/246/out find=all
                   in=dr5qso.fits ra=RA dec=DEC radius=1 out=qso_2mass.fits
</verbatim></dt>
<dd><p>Matches a local catalogue <code>dr5qso.fits</code> against the
    VizieR table <code>II/246/out</code> (the 2MASS Point Source Catalogue).
    The search radius is 1 arcsecond, and all 2MASS sources within the
    radius of each input source are returned.
    </p></dd>

<dt><verbatim>
stilts cdsskymatch cdstable=simbad find=best
                   in=sources.txt ifmt=ascii ra=RAJ2000 dec=DEJ2000 radius=8.5
                   blocksize=1000 icmd=progress omode=topcat
</verbatim></dt>
<dd><p>This finds the closest object in the SIMBAD database within 8.5 arcsec
    for each row of an input ASCII table.
    Uploads are done in blocks of 1,000 rows at a time, and progress
    is displayed on the console.  When the match is complete, the
    result is sent directly to a running instance of TOPCAT.
    </p></dd>

<dt><verbatim>
stilts cdsskymatch in=3XMM_DR4cat_slim_v1.0.fits
                   icmd='select "SC_POSERR &lt; 1 &amp;&amp; SC_EXTENT == 0"'
                   cdstable=B/mk/mktypes
                   ra=SC_RA dec=SC_DEC radius=1.5
                   find=best suffixin=_XMM suffixremote=_MK fixcols=all
                   ocmd='select startsWith(spType_MK,\"G\")'
                   out=xmm_gtype.fits
</verbatim></dt>
<dd><p>This locates XMM-Newton point-like sources identified as being
    of spectral type G.
    It uses the
    <webref url="http://xmmssc-www.star.le.ac.uk/Catalogue/xcat_public_3XMM-DR4.html">3XMM-DR4</webref>
    XMM-Newton serendipitous source catalogue as input.
    The <code>icmd</code> filter selects the objects
    in that catalogue with well-defined point-like positions.
    It then matches them with Skiff's MK spectral classification
    catalogue (B/mk/mktypes in VizieR) and finally filters the
    result to include only those sources identified as being of
    spectral type G.  Thanks to Ada Nebot (CDS) for this example.
    </p></dd>
</dl>
</p>

</subsubsect>

</subsect>

<subsect id="cone" tocleaf="yes">
<subhead><title><code>cone</code>: &cone-purpose;</title></subhead>

<p><code>cone</code> is a utility to execute one of the
"Simple" positional DAL query operations on a remote server:
<webref url="http://www.ivoa.net/documents/latest/ConeSearch.html"
        >Simple Cone Search</webref>,
<webref url="http://www.ivoa.net/documents/SIA/"
        >Simple Image Access</webref> (SIA) or
<webref url="http://www.ivoa.net/documents/SSA/"
        >Simple Spectral Access</webref> (SSA).
The job it does is not very complicated: given a base URL for
a service of one of these types and values for the central position
and radius required, it assembles the query URL in the form required
for the relevant protocol, retrieves the result of the query,
and turns it into a table which can be operated on with the usual
STILTS <ref id="pipes">pipeline operations</ref>.
</p>

&cone-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>cone</code>:
<dl>

<dt><verbatim><![CDATA[
stilts cone serviceurl=http://gaia.ari.uni-heidelberg.de/cone/search
            lon=56.75 lat=24.12 radius=0.8
            out=pleiades.fits
]]></verbatim></dt>
<dd><p>Queries the ARI-Gaia cone search service for
    sources within 0.8 of the given sky position,
    and writes the result to a file.
    </p></dd>

<dt><verbatim><![CDATA[
stilts cone serviceurl=http://gaia.ari.uni-heidelberg.de/cone/search
            lon=56.75 lat=24.12 radius=0.8
            verb=1
            ocmd='sorthead 10 phot_g_mean_mag'
            ocmd='keepcols "source_id ra dec phot_g_mean_mag"'
]]></verbatim></dt>
<dd><p>This does the same basic query as the previous example,
    but post-processes the result so that a limited amount of data
    (source identifier, position and magnitude) for only
    the ten brightest sources is written to the console.
    Since most of the columns are discarded, we specify
    <code>verb=1</code> which indicates to the service that only
    a minimal column set is required in the query result.
    </p></dd>

<dt><verbatim><![CDATA[
stilts -verbose
       cone servicetype=ssa serviceurl='http://archive.eso.org/ssap'
            lon=0 lat=90 radius=1.0 skysys=galactic
            omode=count
]]></verbatim></dt>
<dd><p>Queries the ESO Simple Spectral Access service for spectra
    within one degree of the northern galactic pole.
    The <code>omode=count</code> parameter means that it just counts
    the rows and columns and prints the numbers to the console.
    The <code>-verbose</code> flag means that (amongst other things)
    the full URL that the command used to make the query
    will be logged to the console.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="coneskymatch" tocleaf="yes">
<subhead><title><code>coneskymatch</code>:
                &coneskymatch-purpose;</title></subhead>

<p><em>Note: this command is very inefficient for large tables,
and in most cases
<ref id="cdsskymatch"><code>cdsskymatch</code></ref> or
<ref id="tapskymatch"><code>tapskymatch</code></ref>
provide better alternatives.
</em></p>

<p><code>coneskymatch</code> is a utility which performs a 
cone search-like
query to a remote server for each row of an input table.
Each of these queries returns a table with one row for each
item held by the server in the region of sky represented by
the input row.
The results of all the queries are then concatenated into one big
output table which is the output of this command.
</p>

<p>The type of virtual observatory service queried is determined by the 
<code>servicetype</code> parameter.  
Typically it will be a 
<webref url="http://www.ivoa.net/Documents/latest/ConeSearch.html"
        >Cone Search</webref> service,
which queries a remote catalogue for astronomical objects or sources
in a particular region.
However, you can also query 
<webref url="http://www.ivoa.net/Documents/latest/SIA.html"
        >Simple Image Access</webref> and
<webref url="http://www.ivoa.net/Documents/latest/SSA.html"
        >Simple Spectral Access</webref> services
in just the same way, to return tables of available 
image and spectral resources in the relevant regions.
</p>

<p>The identity of the server to query is given by the
<code>serviceurl</code> parameter.
Some advice about how to locate URLs for suitable services is given
in <ref id="coneService"/>.
</p>

<p>The effect of this command is like doing a positional crossmatch 
where one of the catalogues is local and the other is remote and
exposes its data via a cone search/SIA/SSA service.
Because of both the network communication and the necessarily naive 
crossmatching algorithm (which scales linearly with the size of the
local catalogue) however, 
it is only suitable if the local catalogue has a reasonably small 
number of rows, unless you are prepared to wait a long time.
</p>

<p>The <code>parallel</code> parameter allows you to perform multiple
cone searches concurrently, so that instead of completing the 
first cone search, then the second, then the third, 
the program can be executing a number of them at once.
This can speed up operation considerably, especially
in the face of network latency, but beware that submitting a very large
number of queries simultaneously to the same server may overload it,
resulting in some combination of failed queries, ultimately slower runtimes,
and unpopularity with server admins.
Best to start with a low parallelism and cautiously increase it to
see whether there are gains in performance.
</p>

<p>Note that when running, <code>coneskymatch</code> can generate a lot
of WARNING messages.  Most of these are complaining about badly formed
VOTables being returned from the cone search services.  STILTS does its
best to work out what the service responses mean in this case, 
and usually makes a good enough job of it.
</p>

<p><em>Note: this task was known as <code>multicone</code> in its experimental
form in STILTS v1.2 and v1.3.</em>
</p>

&coneskymatch-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>coneskymatch</code>:
<dl>

<dt><verbatim>
stilts coneskymatch serviceurl=http://archive.stsci.edu/hst/search.php \
                    in=messier.xml sr=0.05 out=matches.xml
</verbatim></dt>
<dd><p>This queries the HST cone search service from Space Telescope for
    records within .05 degrees of each Messier object contained in a 
    local VOTable <code>messier.xml</code>.
    The sky positions in the input catalogue are guessed from the available
    table metadata.
    The result is written to a new VOTable, <code>matches.xml</code>.
    Since the <code>servicetype</code> parameter is not given, the 
    default (cone search) service type is assumed.
    </p></dd>

<dt><verbatim>
stilts coneskymatch
       servicetype=sia \
       serviceurl=http://irsa.ipac.caltech.edu/cgi-bin/2MASS/IM/nph-im_sia?type=ql&amp;ds=asky \
       in=messier.xml ra=RA dec=DEC \
       dataformat=image/fits \
       out=fitsimages.xml
</verbatim></dt>
<dd><p>This is similar to the previous example, but 
    instead of querying an HST cone search server for catalogue objects
    near the input table positions, it queries a 2MASS Simple Image Access
    (SIA) server for images.  
    It also explicitly names the columns holding the J2000 positions of
    reach record in the input catalogue as <code>RA</code> and <code>DEC</code>.
    The search radius parameter (<code>sr</code>) is not set here;
    for SIA queries the default search radius is zero, which has the
    special meaning of including any image which covers the requested position.
    Setting <code>dataformat=image/fits</code> (which is the default) 
    requests only records describing FITS-format images to be returned;
    setting it to an empty value might return other formats such as JPEG too.
    </p></dd>

<dt><verbatim>
stilts coneskymatch \
       serviceurl='http://www.nofs.navy.mil/cgi-bin/vo_cone.cgi?CAT=NOMAD' \
       in=vizier.xml#7 \
       icmd='addskycoords -inunit sex fk4 fk5 RAB1950 DEB1950 RAJ2000 DEJ2000' \
       icmd='progress'
       ra=RAJ2000 dec=DEJ2000 sr=0.01 \
       ocmd='replacecol -units deg RA hmsToDegrees(RA[0],RA[1],RA[2])' \
       ocmd='replacecol -units deg DEC dmsToDegrees(DEC[0],DEC[1],DEC[2])' \
       omode=topcat
</verbatim></dt>
<dd><p>In this example some pre-processing of the input catalogue and
    post-processing of the output catalogue is performed as well as the
    multiple cone search itself.
    </p>
    <p>The input catalogue, which is the 8th TABLE element in a VOTable file,
    contains sky positions in sexagesimal FK4 (B1950) coordinates.  
    The <code>icmd=addskycoords...</code> parameter specifies a filter which
    will add new columns in FK5 (J2000) degrees, which are what the 
    <code>coneskymatch</code> command requires.
    The <code>icmd=progress</code> parameter specifies a filter which
    will write progress information to the terminal so you can see how
    the queries are progressing.
    </p>
    <p>The NOMAD service specified by the <code>serviceurl</code> parameter
    used here happens to return results with the RA/DEC columns represented
    in a rather eccentric format, namely 3-element floating point arrays 
    representing (hours,minutes,seconds)/(degrees,minutes,seconds).
    The two <code>ocmd=replacecol...</code> filters replace the values of
    these columns with the scalar equivalents in degrees.
    Finally, the <code>omode=topcat</code> parameter causes the result
    table to be loaded directly into TOPCAT (if it is available).
    </p>
    </dd>

<dt><verbatim>
stilts coneskymatch serviceurl='http://archive.stsci.edu/iue/search.php?' \
                    in=queries.txt ifmt=ascii \
                    ra='&column.id.char;1' dec='&column.id.char;2' \
                    sr='&column.id.char;3' copycols='&column.id.char;4' \
                    out=found.fits
</verbatim></dt>
<dd><p>Here the input is a plain text table with four unnamed columns, 
    giving in order the right ascension, declination,
    positional error and name of target objects.
    The command carries out a cone search to the named service for
    each one.  Note in this case the search radius (<code>sr</code> parameter)
    is taken from the table and so varies for each query.
    The <code>copycols</code> parameter has the value 
    '<code>&column.id.char;4</code>',
    which means that the value of the fourth column of the input table 
    will be prepended to each row of the output table for 
    which it is responsible.
    Output is to a FITS table.
    </p></dd>
</dl>
</p>

</subsubsect>

<subsubsect id="coneService">
<subhead><title>Locating Cone Query Service URLs</title></subhead>

<p>To use the <code>coneskymatch</code> command you need the
<label>service URL</label> (also known as the <label>base URL</label>
or <label>access URL</label>) of a cone search, SIA or SSA service to use.
If you know one of these representing a service that you wish to use, 
you can use it directly.
</p>

<p>If you don't, you will need to find the URL from somewhere.
It is the job of the Virtual Observatory <label>Registry</label>
to keep a record of where you can find various astronomical services,
so this is where you should look.
</p>

<p>There are various ways you can interrogate the registry; the
easiest is probably to use a graphical registry search tool.
One such tool is 
<webref url="http://www.astrogrid.org/">AstroGrid</webref>'s 
<label>VOExplorer</label>, which allows you to perform sophisticated searches 
for cone search, SIA or SSA services.
Another option is to use
<webref url="http://www.starlink.ac.uk/topcat/">TOPCAT</webref>;
the 
<webref url="http://www.starlink.ac.uk/topcat/sun253/ConeSearchDialog.html"
        >Cone Search</webref>,
<webref url="http://www.starlink.ac.uk/topcat/sun253/SiapTableLoadDialog.html"
        >SIA</webref> and
<webref url="http://www.starlink.ac.uk/topcat/sun253/SsapTableLoadDialog.html"
        >SSA</webref>
load dialogues allow you to search the registry for these services 
prior to performing a query;
you can just use the registry part and cut'n'paste the URL which
is shown.
</p>

<p>Other registry querying tools are available, including STILTS's
<ref id="regquery" plaintextref="yes"><code>regquery</code></ref>
command.  See that section of the manual for details, but for
instance to locate registered Cone Search services which have
something to do with SDSS data, you could execute the following:
<verbatim>
    stilts regquery query="capability/@standardID = 'ivo://ivoa.net/std/ConeSearch' and title like '%SDSS%'" \
           ocmd="keepcols 'shortName AccessUrl'" \
           ofmt=ascii
</verbatim>
Writing just 
<code>query="capability/@standardID = 'ivo://ivoa.net/std/ConeSearch'"</code>
with no further qualification would give you <em>all</em> 
registered cone search services.
</p>

</subsubsect>

</subsect>

<subsect id="datalinklint" tocleaf="yes">
<subhead><title><code>datalinklint</code>:
                &datalinklint-purpose;</title></subhead>

<p><code>datalinklint</code> runs a series of tests on a VOTable
that is supposed to conform to the <em>{links}-response</em> format
defined in the IVOA
<webref url="http://www.ivoa.net/documents/DataLink/">DataLink</webref>
specification, and reports the results.
This is not likely to be of use to normal users,
it is intended for people developing or operating DataLink-compliant
services to assess correctness, perhaps with a view to improving
compliance.
</p>

<p>To run it, you point it at the URL or filename of a VOTable of interest,
and it runs various tests on it and reports them to standard output.
Only the input table itself, and if the input location uses HTTP the
HTTP headers, are tested, this validator does not inspect any other
resources.
As well as validation checks, some reporting of the table's content
(such as a summary of the link defined by each row and a listing of
the defined service descriptors) is provided as INFO-level output
for convenience.  This can be suppressed if preferred by use of
the <code>report</code> parameter.
</p>

<p>This operates in much the same way as
the <ref id="taplint"><code>taplint</code></ref> command,
and the output has a similar format, though unlike
<code>taplint</code> this command does not divide the testing up
into stages.
Each report line is of the form:
<verbatim>
   T-MMMMxN aaaaa...
</verbatim>
where the parts have the following meanings:
<ul>
<li><code>T</code>: Report type,
    one of E(rror), W(arning), I(nfo), S(ummary), F(ailure).
    See the documentation of the <code>report</code> parameter for
    further description of what these mean.
    The <code>report</code> parameter can be used to suppress some of these;
    only <code>E</code> indicates actual service compliance errors, but
    including the others may make it easier to see what's going on.
    </li>
<li><code>MMMM</code>: Message label, which is always the same for
    messages generated by the same test, is usually different for
    messages generated by different tests, and may be somewhat mnemonic.
    </li>
<li><code>x</code>: Continuation indicator,
    either "<code>-</code>" or "<code>+</code>".
    In most cases it is "<code>-</code>", indicating the first line of
    a message, but multi-line messages (rare) use "<code>-</code>" for the first
    line and "<code>+</code>" for any continuation lines.
    </li>
<li><code>N</code>: Sequence number, which is 1 for the first time
    message <code>T-MMMM</code> is reported, and increases by one
    for each subsequent appearance.
    After a certain maximum (determined by the <code>maxrepeat</code>
    parameter) additional reports with the same code are no longer output
    individually, but a summary of the number of reports so discarded
    is written at the end of the section with the character "<code>x</code>"
    instead of the sequence number.
    This behaviour prevents the output being swamped by
    multiple reports of the same issue.
    If the <code>maxrepeat</code> parameter is increased above 9, more than
    one digit will be used here (so e.g. for maxrepeat=999, the format would
    be <code>NNN</code> not <code>N</code>).
    </li>
<li><code>aaaaa...</code>: Message text, a free text description of
    what is being reported.
    </li>
</ul>
</p>

<p>If you don't like that format, others may be selected using the
<code>format</code> parameter, which currently also supports JSON.
For more flexible interaction with the output you can invoke
<code>datalinklint</code> <ref id="taskApi">programmatically</ref>
and supply your own 
<javadoc docset="&stiltsjavadocs;"
         class="uk.ac.starlink.ttools.taplint.OutputReporter"/>
instance.
</p>

<p>As with any validator, this command does not guarantee to pick up
everything wrong with the indicated VOTable, but it tries as hard
as it can to check against the rules set out in the DataLink
specification and other related documents as appropriate.
A non-exhaustive list of the items checked is:
<ul>
<li>VOTable validation <em>&#xe0; la</em>
    <ref id="votlint"><code>votlint</code></ref></li>
<li>Correct RESOURCE structure of VOTable document</li>
<li>All required columns are present with mandated metadata</li>
<li>Optional but documented columns have mandated metadata if present</li>
<li>Each row contains one of <code>access_url</code>,
    <code>service_def</code>, <code>error_message</code></li>
<li>Service descriptors correctly specified and referenced</li>
<li>Syntax of <code>error_message</code> column values</li>
<li>Syntax of <code>content_type</code> column values</li>
<li>Syntax of <code>semantics</code> column values</li>
<li>Values of <code>content_qualifier</code> column values if present</li>
<li>Syntax of <code>link_auth</code> column values if present</li>
<li>Results for the same ID are contiguous (DataLink 1.1)</li>
<li>StandardID INFO is present and correct (Datalink 1.1)</li>
<li>Reported HTTP content-type, if applicable</li>
<li>Correct serialization (TABLEDATA) is used</li>
<li>DALI error-response checking, if applicable</li>
</ul>
At present <strong>no checking</strong> is performed on the link targets;
tests are confined to the presented document itself.
</p>

<p>HTTP connections made by this validator are flagged in the
<code>User-Agent</code> field with the token
"<code>(IVOA-test)</code>".
</p>

&datalinklint-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>datalinklint</code>:

<dl>

<dt><verbatim><![CDATA[
stilts datalinklint votable=links-response.vot
]]></verbatim></dt>
<dd><p>Perfoms validation on a VOTable file that has been downloaded
    from a DataLink links service, and reports the results to standard
    output.
    </p></dd>

<dt><verbatim><![CDATA[
stilts datalinklint votable=http://example.org/links?ID=2112
                    truncate=80 report=EW
]]></verbatim></dt>
<!-- 'http://dc.g-vo.org/califa/q3/dl/dlmeta?'
     'ID=ivo://org.gavo.dc/~?califa/datadr3/COMB/ARP220.COMB.rscube.fits' -->
<dd><p>Validates a links response directly from the server,
    testing the HTTP transport requirements as well as the content.
    Output lines are truncated to 80 characters for convenience,
    and only the E(rror) and W(arning) lines are displayed.
    </p>
    <p>The output of this invocation might look like:
    <verbatim><![CDATA[
This is the STILTS DataLink validator, 3.1-3
E-DLCT-1 Incorrect Content-Type text/xml for DataLink service http://example....
E-RUCD-1 Wrong UCD for column service_def; meta.code != meta.ref
E-SMCO-1 Unknown predicate 'alternative' from core DataLink vocabulary at row 1
E-SMCO-2 Unknown predicate 'alternative' from core DataLink vocabulary at row 2

Totals: Errors: 4; Warnings: 0
    ]]></verbatim>
    To output in JSON format instead, you could specify
    <code>format=json</code>.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="funcs" tocleaf="yes">
<subhead><title><code>funcs</code>: &funcs-purpose;</title></subhead>

<p><code>funcs</code> is a utility which allows you to browse the
functions you can use in STILTS's algebraic expression language.
Invoking the command causes a window to pop up on the display with
two parts.  The left hand panel contains a tree-like representation of the
functions available - the top level shows the classes (categories)
into which the functions are divided, and if you open these up
(by double clicking on them) each contains a list of functions and
constants in that class.
If you click on any of these classes or their constituent functions
or constants, a full descritption of what they are and how to
use them will appear in the right hand panel.
</p>

<p>The information available from this command is the same as that given
in <ref id="staticMethods"/>, but the graphical
browser may be a more convenient way to view the documentation.
There are no parameters.
</p>

&funcs-summary;

</subsect>

<subsect id="pixfoot" tocleaf="yes">
<subhead><title><code>pixfoot</code>: &pixfoot-purpose;</title></subhead>

<p><code>pixfoot</code> takes a list of sky positions from an input
table and generates a pixel map describing a sky region which includes
them all.
Currently the output is to a format known as a 
<webref url="http://www.ivoa.net/Documents/MOC/"
        >Multi-Order Coverage map</webref> (MOC),
which is a HEALPix-based format composed of a list of HEALPix pixels
of different sizes, which can efficiently describe complex regions.
Other output formats may be introduced in the future.
</p>

<p>See also the
<ref id="Coverage">Coverage</ref> class
for MOC-related functions.
</p>

&pixfoot-summary;

<subsubsect id="pixfoot-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>pixfoot</code>:
<dl>

<dt><verbatim>
stilts pixfoot in=survey.vot order=8 mocfmt=fits out=sfoot.fits
</verbatim></dt>
<dd><p>Generates an order-8 FITS MOC file from the point positions of
    rows in the given VOTable.  The columns representing sky position
    are determined automatically (if possible)
    by examining the metadata in the input table.
    </p></dd>

<dt><verbatim>
stilts pixfoot in='jdbc:mysql://localhost/astro1#SELECT * FROM first1'
               icmd='addskycoords galactic icrs GLON GLAT ALPHA DELTA'
               ra=ALPHA dec=DELTA radius=20./3600.
               order=13 mocfmt=fits out=first.moc
</verbatim></dt>
<dd><p>Generates an order-13 FITS MOC file from positions in a table
    held in a database.  The positions in the original table are in
    galactic coordinates, so have to be converted to equatorial (ICRS) first.  
    The map is formed in this case by surrounding each point by a
    disc of 20 arcsec.
    Note that JDBC database access will have to be set up as per
    <ref id="jdbcConfig"/> for this command to work.
    </p></dd>

</dl>
</p>

</subsubsect>
</subsect>


<subsect id="pixsample" tocleaf="yes">
<subhead><title><code>pixsample</code>: &pixsample-purpose;</title></subhead>

<p><code>pixsample</code> samples data at the sky position 
represented by each row from an all-sky map contained
in a HEALPix-format pixel data file.
Such files are actually tables (usually in FITS format)
in which the row number corresponds to
a HEALPix pixel index, and the pixel values are cell contents;
one or more columns may be present containing values for one or more
all-sky maps.  The result of this command is to add a column to
the input table representing the pixel data at the position of
each input row for each of the data columns in the
HEALPix table.
</p>

<p>This command does not attempt to convert between coordinate systems
except as instructed, so it is important to know what coordinate
system the HEALPix file is in, and ensure that the coordinates
supplied from the input table match this.
You may need to examine the documentation or headers of the HEALPix
file in question to find out.
See the <ref id="pixsample-examples">Examples</ref> section for
some examples.
</p>

<p>There is a choice of how the sampling is done;
the simplest way is just to use the value of the pixel covering 
the indicated position.  An alternative is to average over a
disc of given radius (perhaps a function of the input row).
Other options (e.g. max/min) could easily be added.
</p>

<p>Although HEALPix is not a common format for storing image
data in general, it is used for storing a number of important all-sky
data sets such as the WMAP results and Schlegel dust maps.
The NASA 
<webref url="https://lambda.gsfc.nasa.gov/" plaintextref="yes">LAMBDA</webref>
(Legacy Archive for Microwave Background Data Analysis)
archive has a number of maps in a suitable format,
including foreground data like predicted reddening as well as CMB maps.
</p>

<p>Note at present this command only supports all-sky, not partial,
HEALPix maps. Partial map support may be added at some point in
the future if there is demand.
</p>

&pixsample-summary;

<subsubsect id="pixsample-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>pixsample</code>:
<dl>

<dt><verbatim>
stilts pixsample in=szdata.fits pixdata=wmap_ilc_7yr_v4.fits
                 lat=GAL_LAT lon=GAL_LON pcmd='keepcols TEMPERATURE'
                 out=szdata_cmb.fits
</verbatim></dt>
<dd><p>Samples from a HEALPix file containing WMAP data are added to an
    input file <code>szdata.fits</code>, giving an output file
    <code>szdata_cmb.fits</code> which is the same but with an additional
    column <code>TEMPERATURE</code>.
    The sampling is done using the default statistical mode <code>point</code>,
    which just takes a point sample at the input position.
    The HEALPix file must have its pixels ordered using galactic
    coordinates, since that is the coordinate system available from
    the input table.
    </p>
    <p>
    The pixdata file used here can be found (at time of writing) at
    <webref
    url='https://lambda.gsfc.nasa.gov/data/map/dr4/dfp/ilc/wmap_ilc_7yr_v4.fits'
    /> (24 Mbyte).
    </p></dd>

<dt><verbatim>
stilts pixsample in=messier.xml pixdata=lambda_sfd_ebv.fits
                 stat=mean radius=5./60.
                 insys=icrs pixsys=galactic lon=RA2000 lat=DEC2000
</verbatim></dt>
<dd><p>Samples data from a HEALPix table, averaging over a sampling radius
    of 5 arcmin.  The coordinates in the input table are only available
    as ICRS (RA,Dec) coordinates, and the arrangement of the HEALPix pixels
    in the pixel data file uses galactic coordinates
    (you can only determine this by looking at the FITS headers or
    documentation of that file), so it is necessary to use the
    <code>insys</code> and <code>pixsys</code> parameters for conversion.
    </p>
    <p>
    The pixdata file used here can be found (at time of writing) at
    <webref
     url='https://lambda.gsfc.nasa.gov/data/foregrounds/SFD/lambda_sfd_ebv.fits'
    /> (25 Mbyte).
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2plane" tocleaf="yes">
<subhead><title><code>plot2plane</code>: &plot2plane-purpose;</title></subhead>

<p><code>plot2plane</code> draws plots on a Cartesian 2-dimensional surface.
</p>

<p>Positional coordinates are specified as <code>x</code>, <code>y</code> pairs,
e.g.:
<verbatim>
   plot2plane layer1=mark in1=cat.fits x1=RMAG y1=RMAG-BMAG
</verbatim> 
</p>

&plot2plane-layers;

&plot2plane-summary;

<subsubsect id="plot2plane-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plot2plane</code>:
<dl>

<dt><verbatim>
stilts plot2plane yflip=true layer_1=mark in_1=cat.fits x_1=BMAG-RMAG y_1=BMAG 
</verbatim></dt>
<dd><p>This is a colour-magnitude diagram where the input table
    has columns named RMAG and BMAG.
    The Y axis is inverted so that the magnitude values increase
    downwards not up.
    The plot is displayed in a window on the screen, and may be
    panned and zoomed with the mouse.
    </p></dd>

<dt><verbatim>
stilts plot2plane layer=histogram in=hip_main.fits x=plx xlog=true
                  xlabel=Parallax ylabel=
</verbatim></dt>
<dd><p>Plots a histogram of parallaxes for Hipparcos data,
    with a logarithmic X axis.  The axes are labelled explicitly,
    with an empty string in the case of the Y axis.
    </p></dd>

<dt><verbatim>
stilts plot2plane xpix=600 ypix=500
                  in=gavo_g2.fits x=X y=Y
                  shading=aux aux='atan2(vely,velx)' auxmap=hue auxvisible=false
                  layer_m=mark shape_m=cross size_m=4
                  layer_v=xyvector xdelta_v=velx ydelta_v=vely scale_v=2
                  out=velocities.pdf
</verbatim></dt>
<dd><p>Two layers are plotted, point markers representing position
    (4 pixels radius, shaped like crosses)
    and vectors representing velocity.
    Both markers and vectors are coloured according to the direction
    (arctan(vely/velx)) of the arrows,
    so it's easy to see points moving in similar directions;
    the "hue" colour map is good for this, since it's periodic,
    so values of +Pi and -Pi have the same colour.
    Since it's not very revealing in this case, display of the aux
    axis colour ramp beside the plot has been turned off.
    Since the X and Y coordinates and the colouring is common to both
    layers, the relevant parameters can given without suffixes to avoid
    having to repeat them.
    Output is to a PDF file.
    </p></dd>

<dt><verbatim>
stilts plot2plane xmin=0 xmax=6.283 ymin=-1 ymax=1 xlabel=Time
                  layer=function axis=horizontal xname=time fexpr='sin(time)'
                  dash=3,2 thick=4 color=ee6aa7
</verbatim></dt>
<dd><p>Plots a sine curve to the screen.  Initially the view is of one
    period, but you can pan and zoom interactively to see any range.
    The line is plotted in hot pink, four pixels wide, with a custom dash
    pattern.
    Since the <ref id="layer-function">function</ref> layer type has no
    data coordinates, no input table is required.
    The layer suffix here is the empty string; since there's only one
    layer, it doesn't cause any problems.
    </p></dd>

<dt><verbatim>
stilts plot2plane ylog=true xflip=true xmin=-5.2 xmax=3.8 ymin=250 ymax=3.5e5
           in1=6dfgs_E7.fits x1=bmag-rmag y1=vel
           layer1a=mark color1a=cyan
           layer1b=contour color1b=yellow smooth1b=9 scaling1b=log
           layer1c=mark icmd1c='every 35;select star'
                        shape1c=filled_triangle_down size1c=5 color1c=red
                        shading1c=transparent opaque1c=3
           layer2=function fexpr2='exp(x*2+12)' color2=black antialias2=true
                           dash2=dash thick2=3
           leglabel1a=Population leglabel1c=Sample legpos=.95,.95 legseq=1a,1c
           fontsize=16 texttype=latex ylabel="v\,/\,km.s^{-1}" xlabel=colour
</verbatim></dt>
<dd><p>There are four layers: 1a, 1b and 1c use the same positional data
    from the same input file, so the positional coordinates common to them
    are given the suffix "1".
    Layer "2" is unrelated, and has no input data, since it's just an
    analytic function.
    The legend is positioned to taste, and its content
    is manipulated so that only datasets 1a and 1c are described,
    and they are given custom names (the default would be their suffix names).
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2sky" tocleaf="yes">
<subhead><title><code>plot2sky</code>: &plot2sky-purpose;</title></subhead>

<p><code>plot2sky</code> draws plots on the celestial sphere.
This can be represented in a number of ways, controlled by the
<code>projection</code> parameter; by default the view is of a rotatable
sphere seen from the outside
(which approximates to a tangent projection for small regions of the sky),
but Hammer-Aitoff and Plate Car&#x00e9;e projections are also available.
A number of options are also provided for drawing and labelling the
grid showing celestial coordinates.
</p>

<p>Positional coordinates are specified as <code>lon</code>, <code>lat</code>
pairs giving longitude and latitude in decimal degrees.
By default these are represented in the output in the same, unlabelled,
coordinate system.
However the command can can also transform between different coordinate
systems if you specify the
(per-plot) view system with the <code>viewsys</code> parameter and
(per-layer) data system with the <code>datasysN</code> parameter,
e.g.:
<verbatim>
   plot2sky viewsys=galactic
            layer1=mark in1=cat.fits lon1=RA2000 lat1=DEC2000 datasys1=equatorial
</verbatim>
</p>

&plot2sky-layers;

&plot2sky-summary;

<subsubsect id="plot2sky-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plot2sky</code>:
<dl>

<dt><verbatim>
stilts plot2sky in=messier.xml lon=RA lat=DEC
                layer.pos=mark size.pos=4
                layer.txt=label label.txt=Name color.txt=slategrey
</verbatim></dt>
<dd><p>Plots the positions of all the Messier objects on the sky,
    with text labels giving their object names.
    This displays a sphere on the screen that you can rotate/zoom using
    the mouse.
    </p></dd>


<dt><verbatim>
stilts plot2sky projection=aitoff
                xpix=600 ypix=300
                gridcolour=green labelcolour=black
                fontsize=10 gridaa=true texttype=antialias
                sex=true crowd=4
</verbatim></dt>
<dd><p>This just plots a celestial coordinate grid with no data.
    Various options are tweaked to adjust the appearance of the grid.
    </p></dd>

<dt><verbatim>
stilts plot2sky xpix=1000 ypix=500 fontsize=18 crowd=2
                projection=aitoff viewsys=galactic
                layer1=mark size1=0
                shading1=density densemap1=gnuplot2 densefunc1=log
                densesub1=0.5,.95 denseclip1=0.02,1
                in1=gums_mw_all.fits
                lon1=alpha lat1=delta datasys1=equatorial 
                icmd1=progress out=mw.pdf
</verbatim></dt>
<dd><p>Makes an all-sky plot using a Hammer-Aitoff projection into
    galactic coordinates of a large dataset.
    <ref id="shading-density">Density</ref> shading
    means that the colour at each point is dependent on
    how many points are plotted; the density colour map has been fine-tuned
    here to get a specific visual effect.
    The sky coordinates in the input file (alpha and delta) are equatorial,
    but these are transformed to galactic coordinates for plotting.
    The <ref id="progress">progress</ref> filter applied to the input table
    displays a progress indicator on the console to see how far it's got.
    The result is written to a PDF file.
    </p>
    <p>This command was used to plot the GUMS-10 MW dataset, a simulation
    of the milky way stars seen by the Gaia satellite;
    The 2.1 billion row plot took about 45 minutes.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2cube" tocleaf="yes">
<subhead><title><code>plot2cube</code>: &plot2cube-purpose;</title></subhead>

<p><code>plot2cube</code> draws plots in a Cartesian 3-dimensional space.
The plotting volume is a cube,
which is viewed from the outside and usually bounded by
an annotated wire frame.
</p>

<p>Positional coordinates are by default specified as
<code>x</code>, <code>y</code>, <code>z</code> triples,
e.g.:
<verbatim>
   plot2cube layer1=mark in1=sim.fits x1=XPOS y1=YPOS z1=ZPOS
</verbatim>
However other coordinate specifications can be chosen
using the per-layer <code>geomN</code> parameter,
e.g.:
<verbatim>
   plot2cube layer1=mark in=sim.fits geom1=vector xyz1=POS3D
</verbatim>
or
<verbatim>
   plot2cube layer1=mark in1=sphere.fits geom1=polar lon1=RA lat1=DEC r1=RADIUS
</verbatim>
</p>

&plot2cube-layers;

&plot2cube-summary;

<subsubsect id="plot2cube-examples">
<subhead><title>Examples</title></subhead>

<p>Some examples of <code>plot2cube</code> are shown below.
See <ref id="animate"/> for some examples of producing
<strong>animations</strong>, for instance of a rotating cube.
<dl>

<dt><verbatim>
stilts plot2cube
</verbatim></dt>
<dd><p>Just displays a unit cube wireframe in a window.
    You can rotate it with the mouse.
    </p></dd>

<dt><verbatim>
stilts plot2cube layer.1=mark in.1=sim.fits x.1=x y.1=y z.1=z
                 shading.1=density densemap.1=pastel
</verbatim></dt>
<dd><p>Plots markers with x,y,z positions on the screen.
    You can rotate, zoom and pan the cube on the window this produces.
    Density shading is used, which means you can see the lines of sight
    along which most objects fall, though single points are still visible.
    Density shading is usually a good choice if there is just one dataset,
    though it can get confusing with more than one.
    </p></dd>

<dt><verbatim>
stilts plot2cube in=gavo_g2.fits
                 x=X y=Y z=Z
                 shading=aux aux=HALOID opaque=2.5 auxmap=red-blue
                 layer_m=mark shape_m=open_circle size_m=2
                 layer_v=xyzvector xdelta_v=velX ydelta_v=velY zdelta_v=velZ
</verbatim></dt>
<dd><p>Plots points in three dimensions with little arrows representing
    velocity as well as position markers; layer <code>_m</code> draws the
    markers and layer <code>_v</code> draws the arrows.
    Points and vectors are coloured according to the HALOID data value.
    The positional coordinates (x, y, z) and the shading options are
    common to both layers, so they can be specified without a prefix.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2sphere" tocleaf="yes">
<subhead><title><code>plot2sphere</code>:
                &plot2sphere-purpose;</title></subhead>

<p><code>plot2sphere</code> draws plots in an isotropic 3-dimensional space
using spherical polar coordinates.
The plotting volume is a cube,
which is viewed from the outside and usually bounded by a
wire frame annotated by Cartesian coordinates.
This viewing cube is not necessarily centered on the coordinate origin.
</p>

<p>This plotting geometry is like that used by
<ref id="plot2cube"><code>plot2cube</code></ref>,
but the coordinate unit size is always the same in the three dimensions,
and the coordinates are specified differently.
</p>

<p>Positional coordinates are specified as
<code>lon</code>, <code>lat</code>, <code>r</code> triples,
e.g.:
<verbatim>
   plot2sphere layer1=mark in1=survey.fits lon1=RA lat1=DEC r1=REDSHIFT
</verbatim>
</p>

&plot2sphere-layers;

&plot2sphere-summary;

<subsubsect id="plot2sphere-examples">
<subhead><title>Examples</title></subhead>

<p>Some examples of <code>plot2cube</code> are shown below.
See <ref id="animate"/> for some examples of producing
<strong>animations</strong>, for instance of a rotating cube.
<dl>

<dt><verbatim>
stilts plot2sphere in=hip_main.fits lon=radeg lat=dedeg r=plx
                   layer1=mark shading1=density densemap1=cyan-magenta
</verbatim></dt>
<dd><p>Plots points with RA, Dec and parallax coordinates in 3D.
    Density shading is used, which means you can see the lines of sight
    along which most objects fall, though single points are still visible.
    Density shading is usually a good choice if there is just one dataset,
    though it can get confusing with more than one.
    </p></dd>

<dt><verbatim>
stilts plot2sphere in=hip_main.fits lon=radeg lat=dedeg r=plx
                   layer1=mark shading1=density densemap1=cyan-magenta
                   cx=0 cy=0 cz=0 scale=38 texttype=antialias gridaa=true
</verbatim></dt>
<dd><p>The same as the previous example but with some more configuration
    of the axes.  The data origin is placed at the centre of the visible cube
    (this is the position around which the cube will rotate when
    you drag the mouse), and the size of the cube sides in data coordinates
    is set explicitly.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2corner" tocleaf="yes">
<subhead><title><code>plot2corner</code>: &plot2corner-purpose;</title>
         </subhead>

<p><code>plot2corner</code>
represents the relationships between multiple quantities
by drawing a scatter-like plot of every pair of coordinates,
and/or a histogram-like plot of every single coordinate,
and placing these on (half or all of) a square grid.
The horizontal coordinates of all the plots on each column,
and the vertical coordinates of all the plots on each row,
are aligned.
Single-coordinate (histogram-like) plots appear on the diagonal,
and coordinate-pair (scatter plot-like) plots appear off diagonal.
By default only the diagonal and sub-diagonal part of the resulting
plot matrix is shown, since the plots above the diagonal are equivalent
to those below it, but this is configurable.
This representation is variously known as a <label>corner plot</label>,
<label>scatter plot matrix</label>, <label>splom</label> or
<label>pairs plot</label>.
</p>

<p>In principle any number of quantities can be simultaneously compared
in this way, but depending on the output format,
attempting to use too many may make the individual
plots too small to be useful.
</p>

<p>The number D of quantities to compare (the dimensionality of the
space from which you want to plot 2- and 1-dimensional projections)
is given by the <code>nvar</code> parameter.
Each specified layer then requires D positional coordinates,
given by the parameters <code>x1</code>, <code>x2</code>, ... <code>xD</code>.
The resulting grid of plots will have a linear dimension of D
if there are histogram-like layers included,
or D-1 if there are only scatter-plot like layers.
As well as the positional coordinate parameters <code>xK</code> themselves,
some of the other parameters are indexed by the coordinate index <code>K</code>
as well, for instance
<code>xKlog</code>, <code>xKflip</code>,
<code>xKmin</code> and <code>xKmax</code>.
</p>

&plot2corner-layers;

<figure>
<caption><px>Example output from <code>plot2corner</code>.</px></caption>
<figureimage src="&FIG.plot2-matrix;"/>
</figure>

&plot2corner-summary;

<subsubsect id="plot2corner-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>plot2corner</code>:

<dl>

<dt><verbatim>
stilts plot2corner in=:attractor:100000,rampe
                   nvar=3 x1=x x2=y x3=z
                   layer_m=mark
</verbatim></dt>
<dd><p>Plots a grid of scatter plots for a 3-d dataset.
    This presents three plots: X vs. Y, X vs. Z, and Y vs. Z
    in the lower triangle of the grid.
    </p></dd>

<dt><verbatim>
stilts plot2corner in=:attractor:100000,rampe
                   nvar=3 x1=x x2=y x3=z
                   matrixformat=full
                   layer_m=mark layer_h=histogram
</verbatim></dt>
<dd><p>Same as the previous example, but also presents histograms
    (one each for X, Y and Z) along the diagonal.
    Additionally, the <code>matrixformat=full</code> parameter
    means that the full grid (with redundant plots also for
    Y vs. X, Z vs. X, and Z vs. Y) are plotted above the diagonal.
    </p></dd>

<dt><verbatim>
stilts plot2corner in=rrlyrae.fits
                   icmd_A='select best_classification==\"RRAB\"'
                   icmd_C='select best_classification==\"RRC\"'
                   color_A=red color_C=cyan
                   nvar=4 x1=peak_to_peak_g x2=p1 x3=r21_g x4=phi21_g x4min=3
                   layer_A_m=mark layer_C_m=mark
                   layer_A_k=kde layer_C_k=kde smooth_A_k=-50 smooth_C_k=-50
                   layer_f=contour color_f=light_grey smooth_f=10 nlevel_f=6
                   leglabel_A=RRAB leglabel_C=RRC legseq=_A_m,_C_m legpos=1,1
</verbatim></dt>
<dd><p>In this example there are two separate input datasets
    (different selections from the same input table)
    and four quantities to plot.
    One of the coordinates is restricted in range using the
    <code>x4min</code> parameter.
    As well as the <code>mark</code> and <code>kde</code> layers,
    an additional <code>contour</code> layer (of the whole input table)
    is overplotted on the scatter plot grid elements.
    </p></dd>

</dl>

</p>

</subsubsect>

</subsect>

<subsect id="plot2time" tocleaf="yes">
<subhead><title><code>plot2time</code>: &plot2time-purpose;</title></subhead>

<p><code>plot2time</code> draws plots where the horizontal axis represents time.
The time axis can be labelled in various different ways including
MJD, decimal year and ISO-8601 form.
</p>

<p>Positional coordinates are specified as <code>t</code>, <code>y</code> pairs,
with an optional <code>ttype</code> specifier to indicate how the
input value is to be interpreted,
e.g.:
<verbatim>
   plot2time in1=series.fits layer1=line t1=EPOCH ttype1=MJD y1=ENERGY
</verbatim>
</p>

<p>Time values can be represented in various ways in input data,
for instance as Julian Day, Modified Julian Date, decimal years
since 0AD, Unix seconds, ISO-8601, or variants of some of the
above with additional offsets.
In some cases the input format contains enough metadata to
determine how the values should be mapped to a common timescale
(so for instance they can be plotted as MJD or Year/Month/Day),
and in other cases they do not.
For example CDF files and VOTable 1.4 files with <code>TIMESYS</code>
markup contain sufficient metadata, and text inputs using the
ISO-8601 format can usually be identified and understood,
but there's no way to tell automatically whether a numeric column
in a CSV file represents MJD, seconds since a known epoch,
decimal years, or anything else.
For this reason the <code>ttypeN</code> parameter is provided
for all the layer types with a <code>tN</code> coordinate,
as follows:
<dl>
&ttypeN-param;
</dl>
</p>

<p>This command,
unlike the other <code>plot2*</code> commands at time of writing,
can be used to draw <em>multi-zone</em> plots.
These are plots with different panels stacked vertically so that
different datasets can share the same horizontal (time) axis, but have
separate vertical axes, colour maps, legends etc.
The horizontal axes are always synchronized between zones.
This is currently controlled with the <code>zoneN</code> parameter.
For any layer with a layer suffix <code>N</code>, you can specify
a zone identifier as an arbitrary string, <code>Z</code>,
by supplying the parameter <code>zoneN=Z</code>.
Layers with the same value of 
<code>zoneN</code> are plotted in the same zone, and layers with
different values are plotted in different zones.
If no <code>zoneN</code> is given, the layer is assigned to 
a single (unnamed) zone, so with no zone parameters specified
all plots appear in a single zone.
Parameters specific to a given zone can then be suffixed with the
same <code>Z</code> zone identifier.
The <ref id="plot2time-examples">examples</ref> section illustrates
what this looks like in practice.
</p>

<p><em><strong>Note:</strong> The multi-zone feature is experimental.
As currently implemented it lacks some features.
The interface may be changed in a future version.
</em></p>

&plot2time-layers;

&plot2time-summary;

<subsubsect id="plot2time-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plot2time</code>:
<dl>

<dt><verbatim>
stilts plot2time xpix=1000 ypix=300
                 in=ACE_data.vot t=epoch
                 layer.r=line y.r=Br color.r=grey
                 layer.t=line y.t=Bt color.t=cyan 
                 layer.n=line y.n=Bn color.n=pink
</verbatim></dt>
<dd><p>Three time series are plotted on the same axes as lines in different
    colours.  In this case we trust the markup in the VOTable input
    to describe the format of the <code>epoch</code> column.
    </p></dd>

<dt><verbatim>
stilts plot2time xpix=1000 ypix=1000
                 in=ACE_data.vot t=fp_year ttype=decyear
                 layer.r=line y.r=Br zone.r=ZR
                 layer.t=line y.t=Bt zone.t=ZT
                 layer.n=line y.n=Bn zone.n=ZN
                 titleZR="Br" titleZT="Bt" titleZN="Bn"
</verbatim></dt>
<dd><p>The same data is plotted as in the previous example, but in this
    case each line is drawn in a different panel (zone), stacked vertically.
    The default colour is used for each line.
    Each plot is given a different title; note the <code>title</code>
    parameter suffixes refer to the zone identifiers not the layer identifiers.
    The time coordinate in this case is supplied as a decimal year column
    which does not have sufficient metadata to identify it as such,
    so we specify the time format explicitly as <code>decyear</code>.
    </p></dd>

<dt><verbatim>
stilts plot2time tmin=2007-06-07T02:40 tmax=2007-06-07T06:20 tformat=mjd
                 in=STEREO_STA_L1_SEPT_20070607_V05.cdf t=epoch_ns
                 ylabel=Channel
                 layer_3=spectrogram spectrum_3=Spec_0_NS
                 auxmap=accent auxfunc=log
</verbatim></dt>
<dd><p>Plots a spectrogram from a CDF file.
    The range along the horizontal axis is specified explicitly using
    ISO-8601 date strings, but it is labelled in Modified Julian Date.
    </p></dd>

<dt><verbatim>
stilts plot2time in=STEREO_STA_L1_SEPT_20070607_V05.cdf t=epoch_ns
                 layer_1=spectrogram spectrum_1=spec_0_ns zone_1=A
                 layer_2=spectrogram spectrum_2=spec_0_e  zone_2=B
                 layer_3=line y_3='mean(spec_0_ns)' color_3=plum    zone_3=C
                 layer_4=line y_4='mean(spec_0_e)'  color_4=skyblue zone_4=C
                 ylogC=true
                 auxfunc=sqrt auxmapA=viridis auxmapB=magma
</verbatim></dt>
<dd><p>This is a 3-zone plot; zones <code>A</code> and <code>B</code>
    each contains a spectrogram (layers <code>_1</code> and <code>_2</code>),
    and zone <code>C</code> contains two line plots
    (layers <code>_3</code> and <code>_4</code>).
    The Y axis is set to logarithmic for zone C only.
    The colour ramps are configured with <code>aux*</code> parameters;
    the stretch function is set to <code>sqrt</code> for all zones,
    and the colour map is set to different values for zones A and B.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="plot2d" tocleaf="yes">
<subhead><title><code>plot2d</code>: &plot2d-purpose;</title></subhead>

<p><em>This section describes a deprecated command.
It still works, but you are advised to use the more capable
<code><ref id="plot2plane">plot2plane</ref></code> instead.
</em></p>

<p><code>plot2d</code> performs two-dimensional scatter plots,
sending the output to a graphical display or writing it to a file
in some vector or bitmapped graphics format.
You need to supply it with values for one or more X and Y datasets,
in terms of table columns, and it will generate a plot with a point
for each row.
There are many options available to configure the detailed appearance
of the plot, but in its simplest form invocation is quite straightforward.
See <ref id="plot"/> for more discussion on use of the plotting commands.
</p>

&plot2d-summary;

<subsubsect id="plot2d-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plot2d</code> in use:
<dl>
&plot2d-examples;
</dl>
</p>
</subsubsect>

</subsect>

<subsect id="plot3d" tocleaf="yes">
<subhead><title><code>plot3d</code>: &plot3d-purpose;</title></subhead>

<p><em>This section describes a deprecated command.
It still works, but you are advised to use the more capable
<code><ref id="plot2cube">plot2cube</ref></code> or
<code><ref id="plot2sphere">plot2sphere</ref></code> instead.
</em></p>

<p><code>plot3d</code> performs three-dimensional scatter plots,
sending the output to a graphical display or writing it to a file
in some vector or bitmapped graphics format.
You need to supply it with values for one or more X, Y and Z datasets,
in terms of table columns, and it will generate a plot with a point
for each row.
There are many options available to configure the detailed appearance
of the plot, but in its simplest form invocation is quite straightforward.
See <ref id="plot"/> for more discussion on use of the plotting commands.
</p>

&plot3d-summary;

<subsubsect id="plot3d-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plot3d</code> in use:
<dl>
&plot3d-examples;
</dl>
</p>
</subsubsect>

</subsect>

<subsect id="plothist" tocleaf="yes">
<subhead><title><code>plothist</code>: &plothist-purpose;</title></subhead>

<p><em>This section describes a deprecated command.
It still works, but you are advised to use the more capable
<code><ref id="plot2plane">plot2plane</ref></code> instead.
</em></p>

<p><code>plothist</code> performs histogram plots,
sending the output to a graphical display or writing it to a file
in some vector or bitmapped graphics format.
You need to supply it with values for one or more sets of X values,
in terms of table columns, and it will bin the data and draw bars
appropriately.
Plot bounds, bin widths etc may be supplied expliicitly, but will be
calculated from the data and set from defaults as appropriate otherwise.
There are many options available to configure the detailed appearance
of the plot, but in its simplest form invocation is quite straightforward.
See <ref id="plot"/> for more discussion on use of the plotting commands.
</p>

&plothist-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>plothist</code> in use:
<dl>
&plothist-examples;
</dl>
</p>
</subsubsect>

</subsect>

<subsect id="regquery" tocleaf="yes">
<subhead><title><code>regquery</code>: &regquery-purpose;</title></subhead>

<p><code>regquery</code> submits a query to the Virtual Observatory 
<label>registry</label>
and returns the result as a table containing all the records which 
match the condition specified.  The resulting table can be written out 
in any of the supported formats or otherwise processed in the usual ways.
Making use of this command requires an understanding of the
<webref url="http://www.ivoa.net/Documents/latest/VOResource.html"
        >VOResource</webref> schema.
</p>

<p>It is important to note that the results of this command give a
very much flattened and incomplete view of the results of a full 
registry query.  That is because the contents of an IVOA Registry
(see the IVOA 
<webref url="http://www.ivoa.net/Documents/latest/RM.html"
        >Resource Metadata</webref>
and
<webref url="http://www.ivoa.net/Documents/latest/VOResource.html"
        >VOResource</webref>
documents for more detail)
are hierarchical and cannot be faithfully represented in a simple
tabular structure.
Other superior registry search clients exist;
this command is just useful for viewing the results in a rather
simplified way which can be represented as a table.
</p>

&regquery-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>regquery</code>:
<dl>

<dt><verbatim>
stilts regquery query="title like '%IRAS%'" ofmt=ascii out=iras.txt
</verbatim></dt>
<dd><p>Retrieves all the records in the registry whose
    <code>title</code> field contain the string "IRAS".
    The '<code>%</code>' characters function
    as wildcards for the ADQL <code>like</code> operator.
    The output is written to a local ASCII table which can be examined
    later.
    </p></dd>

<dt><verbatim>
stilts regquery query="capability/@standardID = 'ivo://ivoa.net/std/ConeSearch'
                       and curation/@publisher like '%astrogrid%'"
                omode=count
</verbatim></dt>
<dd><p>Searches for all resources which offer a cone search service and
    are published by AstroGrid.
    In this case the records are not stored, but the <code>omode=count</code>
    output mode counts the rows.
    This therefore tells you how many AstroGrid cone search services are
    in the registry.
    </p></dd>

<dt><verbatim>
stilts regquery query="capability/@standardID = 'ivo://ivoa.net/std/SSA'"
                ocmd="keepcols 'identifier accessUrl'"
                ofmt=ascii out=-
</verbatim></dt>
<dd><p>Queries the registry for all Simple Spectral Access services.
       The <code>keepcols</code> filter takes the result and throws away
       all the columns except for <code>identifier</code> and 
       <code>accessUrl</code>, and these are written to the terminal
       int ASCII format.
       </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="server" tocleaf="yes">
<subhead><title><code>server</code>: &server-purpose;</title></subhead>

<p><code>server</code> runs an HTTP server which makes various elements
of STILTS functionality available as HTTP services,
so that they can be run by local or remote clients making HTTP requests
rather than from the more usual command line interface.
</p>

<p>When you run <code>server</code> it will start up a server which 
runs until it is interrupted, and write to standard output
the <em>base URL</em> at which it can be accessed, for instance
"<code>http://localhost:2112/stilts/</code>".
If you point your browser here you will see some examples (hyperlinks to
service requests) of how to use the server.
</p>

<p>See <ref id="serverMode"/> for more discussion of the 
server functionality available.
</p>

<p><strong>Note:</strong>
The <code>server</code> command and associated servlet code are
somewhat experimental.
If you have requirements which are not currently provided, please
contact the author for discussion.
</p>

&server-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of running the <code>server</code> command:
<dl>

<dt><verbatim>
stilts server
</verbatim></dt>
<dd><p>Starts a server on the default port until it is interrupted.
    Most tasks are available in server mode.
    A message will be printed on standard output indicating the base URL
    at which it may be accessed, for instance
    "<code>http://localhost:2112/stilts/</code>".
    </p></dd>

<dt><verbatim>
stilts server port=2100 basepath=tableserv
</verbatim></dt>
<dd><p>Starts a server running on port 2100 with a given URL.
    The URL at which, for instance, the <code>plot2d</code> task 
    can be executed will be 
    "<code>http://</code><em>host</em><code>:2100/tableserv/task/plot2d</code>"
    </p></dd>

<dt><verbatim>
stilts server tasks="plot2d plothist"
</verbatim></dt>
<dd><p>Starts a server with a restricted list of tasks available.
    Only the plotting tasks
    <ref id="plot2d"><code>plot2d</code></ref> and
    <ref id="plothist"><code>plothist</code></ref>
    will be available for execution by clients.
    </p></dd>
</dl>
</p>
</subsubsect>

</subsect>

<subsect id="sqlclient" tocleaf="yes">
<subhead><title><code>sqlclient</code>:
                &sqlclient-purpose;</title></subhead>

<p><code>sqlclient</code> is a simple command-line client 
for use with SQL databases.
One or more SQL statements can be supplied using the <code>sql</code>
parameter.
The result of each statement may be one or more update counts 
(for update-type statements) or tables (for query-type statements).
Tables will be written to
standard output in a format given by the <code>ofmt</code> parameter.
Update results and timing information will be written to standard error.
</p>

<p>In most cases, you will find life easier if you use either the
database's own command-line or GUI client, or, if you require STILTS-type
format conversion or post-processing, a <code>jdbc:</code>-format URL
for the <code>in</code> parameter of the
<ref id="tpipe"><code>tpipe</code></ref> or
<ref id="tcopy"><code>tcopy</code></ref> commands
(see <ref id="jdbcConfig"/> for more explanation of the latter).
However, this command enables you to submit multiple queries over the
same JDBC connection, including ones which do not generate a tabular
result.  It may be useful if a command-line client is not available to
you for the database you are using.
</p>

<p>This command can only be used if you have access to an SQL
database via JDBC.  The details of how to configure a JDBC connection
to a database are discussed in <ref id="jdbcConfig"/> - obviously 
you will need a database to connect to and appropriate permissions 
on it as well as the relevant drivers.
</p>

<p><strong>This command is experimental</strong>,
and it may be enhanced, renamed or withdrawn in future releases.
</p>

&sqlclient-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>sqlclient</code>:
<dl>

<dt><verbatim>
stilts -classpath lib/drivers.jtds-1.1.jar \
       -Djdbc.drivers=net.sourceforge.jtds.jdbc.Driver \
       -Djava.net.preferIPv4Stack=true \
       sqlclient \
           db='jdbc:jtds:sqlserver://amenhotep:1433/twomass' \
           user='guest1' \
           ofmt=csv-nohead \
           sql='SET SHOWPLAN_TEXT ON' \
           sql='SELECT ra,dec FROM twomass_psc WHERE ra BETWEEN 21.7 AND 21.8 \
                                                 AND dec BETWEEN 9.1 AND 9.12'
</verbatim></dt>
<dd><p>This sends two commands to a SQL Server database; the first one
    (SET SHOWPLAN...) sets a flag which causes the DB to return an 
    execution plan rather than the result for subsequent queries, 
    and the second makes the query itself.  
    Since the password is not provided on the command line, a prompt 
    for it will be issued before execution.
    The result is SQL Server's execution plan for the SELECT statement 
    expressed as a headerless comma-separated value table sent to the terminal.
    CSV is chosen for the output format since it does not truncate wide columns.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="sqlskymatch" tocleaf="yes">
<subhead><title><code>sqlskymatch</code>:
                &sqlskymatch-purpose;</title></subhead>

<p><code>sqlskymatch</code> resembles 
<ref id="coneskymatch" plaintextref="yes"><code>coneskymatch</code></ref>,
but instead
of sending an HTTP query to a remote cone search service for each 
match (i.e. each row of the input table), it executes an SQL query
directly.  The query is a SELECT statement with a WHERE clause which
makes restrictions on Right Ascension and Declination columns;
the names of these columns must be given as parameters.
The effect is that of a spatial join between a client-side table and 
a table stored in the database.
</p>

<p>This command can only be used if you have access to an SQL
database via JDBC.  The details of how to configure a JDBC connection
to a database are discussed in <ref id="jdbcConfig"/> - obviously 
you will need a database to connect to and appropriate read permissions 
on it as well as the relevant drivers.
</p>

<p><em>Note: this task was known as <code>sqlcone</code> in its experimental
form in STILTS v1.3.</em>
</p>

&sqlskymatch-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>sqlskymatch</code>:
<dl>

<dt><verbatim>
stilts -classpath lib/drivers/mysql-connector-java.jar \
       -Djdbc.drivers=com.mysql.jdbc.Driver 
       sqlskymatch in=messier.xml ra=RA dec=DEC sr=0.05 \
                   db='jdbc:mysql://localhost/ASTRO1' user=mbt \
                   dbtable=FIRST dbra=_RA2000 dbdec=_DE2000 \
                   out=matches.xml
</verbatim></dt>
<dd><p>This performs a series of SELECT statements on the table FIRST
    in the local MySQL database ASTRO1 to identify database objects in the
    region of each object represented in the VOTable <code>messier.xml</code>.
    The result, a join between the Messier and FIRST tables, is output
    as a VOTable called <code>matches.xml</code>.
    In this case a password has not been supplied on the command line,
    so if one is required it will be prompted for on the console.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="sqlupdate" tocleaf="yes">
<subhead><title><code>sqlupdate</code>: &sqlupdate-purpose;</title></subhead>

<p><code>sqlupdate</code> updates values in an existing table in an SQL
database.  The rows to update are specified, as a normal
SELECT statement, using the <code>select</code> parameter.
Each column to update, and the value to write to it, are given using
the <code>assign</code> parameter.
</p>

<p>Why not just use the database's own UPDATE statement?  In most cases,
that would be a much better idea.  However, using <code>sqlupdate</code>
you can write values using STILTS's 
<ref id="jel" plaintextref="yes">expression language</ref>, and hence take
advantage of its various functions, without having to embed them into
the database.  SQL column names can be used as variables in these expressions,
in the same way that table column names are used as variables in 
other commands such as <code>tpipe</code>.
</p>

<p>This command can only be used if you have access to an SQL
database via JDBC.  The details of how to configure a JDBC connection
to a database are discussed in <ref id="jdbcConfig"/> - obviously 
you will need a database to connect to and appropriate write permissions
on it as well as the relevant drivers.
</p>

<p>This is a somewhat specialised command, and several 
(database/driver-specific) things can go wrong with it.
If you do not have a fairly good understanding of the database with which
you are using it then you may run into problems (but then you'd be unlikely
to have the permissions to do the updates in any case).
</p>

&sqlupdate-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>sqlupdate</code>:
<dl>

<dt><verbatim>
stilts -classpath lib/drivers/mysql-connector-java.jar \
       -Djdbc.drivers=com.mysql.jdbc.Driver \
       sqlupdate db='jdbc:mysql://localhost/RADIO' user=root
       select='SELECT * from FIRST" \
       assign='HTMID=htmIndex(20,POS_EQ_RA,POS_EQ_DEC)'
</verbatim></dt>
<dd><p>Fills in the HTMID column of a table called FIRST in the local MySQL
    database RADIO, using HTM pixel indices based on the existing 
    right ascension and declination columns in that table.
    The HTMID column must exist prior to executing this command.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="taplint" tocleaf="yes">
<subhead><title><code>taplint</code>: &taplint-purpose;</title></subhead>

<p><code>taplint</code> runs a series of tests on a Table Access Protocol
(<webref url="http://www.ivoa.net/Documents/TAP/">TAP</webref>)
service and reports the results.  Unlike most of the other tools in 
this package it is not likely to be of use to normal users;
its intended use is for people developing or operating TAP services
to assess their services, perhaps with a view to improving compliance.
</p>

<p>Testing takes place in a number of stages; it is possible to choose 
which stages are run in by using the <code>stages</code> parameter.
The default output (<code>format=text</code>)
is line-based text to standard output,
and each report line is of the (fairly greppable) form:
<verbatim>
   T-SSS-MMMMxN aaaaa...
</verbatim>
where the parts have the following meanings:
<ul>
<li><code>T</code>: Report type,
    one of E(rror), W(arning), I(nfo), S(ummary), F(ailure).
    See the documentation of the <code>report</code> parameter for
    further description of what these mean.
    The <code>report</code> parameter can be used to suppress some of these;
    only <code>E</code> indicates actual service compliance errors, but
    including the others may make it easier to see what's going on.
    </li>
<li><code>SSS</code>: Stage abbreviation, as used in the <code>stages</code>
    parameter.  The <code>stages</code> parameter can be used to select
    which stages are run.
    </li>
<li><code>MMMM</code>: Message label, which is always the same for
    messages generated by the same test, is usually different for
    messages generated by different tests, and may be somewhat mnemonic.
    </li>
<li><code>x</code>: Continuation indicator,
    either "<code>-</code>" or "<code>+</code>".
    In most cases it is "<code>-</code>", indicating the first line of
    a message, but multi-line messages (rare) use "<code>-</code>" for the first
    line and "<code>+</code>" for any continuation lines.
    </li>
<li><code>N</code>: Sequence number, which is 1 for the first time
    message <code>T-SSS-MMMM</code> is reported, and increases by one
    for each subsequent appearance.
    After a certain maximum (determined by the <code>maxrepeat</code>
    parameter) additional reports with the same code are no longer output
    individually, but a summary of the number of reports so discarded
    is written at the end of the section with the character "<code>x</code>"
    instead of the sequence number.
    This behaviour prevents the output being swamped by
    multiple reports of the same issue.
    If the <code>maxrepeat</code> parameter is increased above 9, more than
    one digit will be used here (so e.g. for maxrepeat=999, the format would
    be <code>NNN</code> not <code>N</code>).
    </li>
<li><code>aaaaa...</code>: Message text, a free text description of
    what is being reported.
    </li>
</ul>
</p>

<p>If you don't like that format, others may be selected using the
<code>format</code> parameter, which currently also supports JSON.
For more flexible interaction with the output you can invoke
<code>taplint</code> <ref id="taskApi">programmatically</ref>
and supply your own 
<javadoc docset="&stiltsjavadocs;"
         class="uk.ac.starlink.ttools.taplint.OutputReporter"/>
instance.
</p>

<p>TAP is a complicated beast, referencing many standards
(including 
<webref url="http://www.ivoa.net/Documents/TAP/">TAP</webref>,
<webref url="http://www.ivoa.net/Documents/UWS/">UWS</webref>,
<webref url="http://www.ivoa.net/Documents/VODataService/"
        >VODataService</webref>,
<webref url="http://www.ivoa.net/Documents/latest/ADQL.html">ADQL</webref>,
<webref url="http://www.ivoa.net/Documents/latest/VOResource.html"
        >VOResource</webref>,
<webref url="http://www.ivoa.net/Documents/VOSI/">VOSI</webref>,
<webref url="http://www.ivoa.net/Documents/TAPRegExt/">TAPRegExt</webref>,
<webref url="http://www.ivoa.net/documents/DALI/">DALI</webref>,
<webref url="http://www.ivoa.net/Documents/ObsCore/">ObsCore</webref>,
<webref url="http://www.ivoa.net/Documents/ObsLocTAP/">ObsLocTAP</webref>,
<webref url="http://www.ivoa.net/Documents/EPNTAP/">EPN-TAP</webref>,
<webref url="http://www.ivoa.net/Documents/VOTable/">VOTable</webref>,
<webref url="http://www.ivoa.net/documents/UCD1+/">UCD</webref>,
<webref url="http://www.ivoa.net/Documents/VOUnits/">VOUnits</webref>,
<webref url="http://www.ivoa.net/documents/SSO/">SSO</webref>,
<webref url="http://www.ivoa.net/documents/Notes/softid/">SoftID</webref>,
<webref url="http://www.w3.org/Protocols/rfc2616/rfc2616.html">HTTP</webref>,
<webref url="https://www.w3.org/TR/rdfa-lite/">RDFa Lite</webref>),
and it is hard to write a validator which is comprehensive, especially
one which can provide useful output for services with a range of 
compliance levels.
This tool tries to make a wide range of tests, but does not claim to
be comprehensive.  An idea of what tests it does perform can be gained 
from the stages listed in the description of the <code>stages</code>
parameter.  It does make a fairly good job of checking that declared
metadata is consistent and matches the data actually returned from queries,
it tests job submission in most of the various ways permitted by the TAP
standard, and it checks all returned VOTables by effectively running them
through <ref id="votlint"><code>votlint</code></ref>.
Things it does not test much include complex ADQL queries, 
coordinate/STC-related data types, queries in non-ADQL languages,
and service registration.
</p>

<p>HTTP connections made by this validator are flagged in the
<code>User-Agent</code> field with the token
"<code>(IVOA-test)</code>".
</p>

&taplint-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>taplint</code>:

<dl>

<dt><verbatim><![CDATA[
stilts taplint http://dc.g-vo.org/tap
]]></verbatim></dt>
<dd><p>Performs a default validation run against the TAP service based
    at the given URL.
    </p></dd>

<dt><verbatim><![CDATA[
stilts taplint tapurl=http://gaia.esac.esa.int/tap-server/tap
               examplesurl=file://localhost/tmp/examples.xml
               stages='TME CAP EXA'
]]></verbatim></dt>
<dd><p>Executes the <code>EXA</code>mples stage against
    the GACS TAP service at ESAC.
    Most of the service endpoints (tables, capabilities, availability etc)
    are found at their default locations
    relative to the given <code>tapurl</code>.
    However, the Examples document is loaded instead
    from the local file at the URL that has been specified by the
    <code>examplesurl</code> parameter.
    This makes is possible to test a non-deployed examples document against
    a deployed TAP service.  It may also be used if certain capabilities
    have been deployed at non-default locations to satisfy multiple security
    models or for other reasons.
    You can play similar tricks with the other <code>*url</code> parameters
    like <code>capabilitiesurl</code> and <code>tablesurl</code>,
    as listed in the <ref id="taplint-usage">documentation</ref>.
    </p>
    <p>The <code>TME</code> (table metadata)
    and <code>CAP</code> (service capabilities)
    stages have been executed along with <code>EXA</code>,
    since taplint needs to pick up the metadata and capabilities in order
    to be able to do some of the checks on the examples it finds.
    If those stage names are not included in the <code>stages</code>
    parameter, the output will include some messages noting that fact,
    and the tests will be less rigorous.
    </p></dd>

<dt><verbatim><![CDATA[
stilts taplint tapurl=http://example.com/tap
               report=EW stages='TMS UWS' truncate=80 maxrepeat=4
]]></verbatim></dt>
<dd><p>A validation run is done against the named TAP service.
    Only Error and Warning type messages are output,
    only two validation stages are performed,
    lines are truncated to a maximum of 80 characters,
    and each message is repeated a maximum of 4 times.
    An invocation like this may be suitable if you find the default
    operation too verbose.
    </p>
    <p>The output of this invocation might look like this:
    <verbatim><![CDATA[
    Section TMS: Check content of tables metadata from TAP_SCHEMA
E-TMS-CINT-1 Column principal in TAP_SCHEMA.columns has wrong type char not int
E-TMS-CINT-2 Column std in TAP_SCHEMA.columns has wrong type char not int
W-TMS-CLUN-1 Unused entry in TAP_SCHEMA.columns table: ivoa.obscore

Section UWS: Test asynchronous UWS/TAP behaviour
E-UWS-GMIM-1 Incorrect Content-Type text/xml != text/plain for http://exampl....
E-UWS-GMIM-2 Incorrect Content-Type text/xml != text/plain for http://exampl....
E-UWS-GMIM-3 Incorrect Content-Type text/xml != text/plain for http://exampl....
E-UWS-GMIM-4 Incorrect Content-Type text/xml != text/plain for http://exampl....
E-UWS-GMIM-x (3 more)

Totals: Errors: 9; Warnings: 1
    ]]></verbatim>
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tapquery" tocleaf="yes">
<subhead><title><code>tapquery</code>: &tapquery-purpose;</title></subhead>

<p><code>tapquery</code> can query remote databases using the
Table Access Protocol (TAP) services by submitting 
Astronomical Data Query Language (ADQL) queries to them and retrieving
the results.
<webref url="http://www.ivoa.net/Documents/TAP/">TAP</webref> and
<webref url="http://www.ivoa.net/Documents/latest/ADQL.html">ADQL</webref>
are Virtual Observatory protocols.
</p>

<p>Queries can be submitted in either synchronous or asynchronous mode,
as determined by the <code>sync</code> parameter.
In asynchronous mode, if the query has not been deleted by the time the 
command exits (see the <code>delete</code> parameter),
the result can be picked up at a later stage using the
<ref id="tapresume"><code>tapresume</code></ref> command.
</p>

<p>Table uploads are supported, so it is possible (if the service supports
this functionality), to upload a local table to the remote database,
perform a query involving it, such as a join with a remote table of some sort, 
and receive the result.  This powerful facility gives you crossmatches
between local and remote tables.
</p>

<p>For services that require authentication, you will need to supply
your credentials as described in <ref id="AuthManager"/>.
Otherwise, queries will be submitted anonymously by default,
but you can force a login attempt to services with optional
authentication by setting <code>auth=true</code>.
</p>

<p>This command does not provide any facility for querying the service
for either table or capability metadata, so you will need to know about
the service capabilities and database structure from some other source
(possibly TOPCAT).
</p>

&tapquery-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tapquery</code>:
<dl>

<dt><verbatim>
stilts tapquery tapurl=https://gea.esac.esa.int/tap-server/tap
                adql='SELECT TOP 1000 source_id, ra, dec
                      FROM gaiadr3.gaia_source
                      ORDER BY random_index'
                out=g.fits
</verbatim></dt>
<dd><p>Executes the given ADQL query on the service referenced by the URL
    and writes the result to a FITS file.
    </p></dd>

<dt><verbatim>
stilts tapquery tapurl=https://gea.esac.esa.int/tap-server/tap
                adql='SELECT TOP 1000 source_id, ra, dec
                      FROM gaiadr3.gaia_source
                      ORDER BY random_index'
                sync=true auth=true
                out=g.fits
</verbatim></dt>
<dd><p>Same as the previous example, except that
    <code>sync=true</code> runs the query synchronously
    instead of asynchronously, and
    <code>auth=true</code> will attempt to log in to the service
    (prompting you for username and password) if it offers authentication.
    </p></dd>

<dt><verbatim><![CDATA[
stilts tapquery
       tapurl=http://dc.g-vo.org/tap
       adql='SELECT *
             FROM twomass.data AS t
             JOIN TAP_UPLOAD.up1 AS s
                  ON DISTANCE(t.raj2000,t.dej2000,s.ra2000,s.dec2000)<5./3600'
       nupload=1 upload1=6dfgs_E7.fits ucmd1='select BMAG-RMAG<0'
       maxrec=20000
       ocmd='tablename 2mass_x_6df' omode=topcat
]]></verbatim></dt>
<dd><p>The local table <code>6dfgs_E7</code> is filtered to contain only
    rather blue objects, and the resulting selection is uploaded to the
    TAP server.  A positional crossmatch with 5 arcsec tolerance 
    is then performed on the server between this uploaded table and the
    <code>twomass.data</code> table held by the service.
    The adjusted <code>maxrec</code> parameter ensures that the result
    will not be artificially truncated to shorter than 20000 rows
    (assuming the service limits permit this).
    When the result is received, it is loaded directly into TOPCAT with
    the name "2mass_x_6df".
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tapresume" tocleaf="yes">
<subhead><title><code>tapresume</code>: &tapresume-purpose;</title></subhead>

<p><code>tapresume</code> can resume monitoring and data retrieval 
from an asynchronous Table Access Protocol query which has already
been submitted.
TAP is a Virtual Observatory protocol.
Such a pre-existing query may have been submitted
by the <ref id="tapquery"><code>tapquery</code></ref> command or
by some completely different mechanism.
It essentially does the same job as <code>tapquery</code> but without
the job submission stage.
It waits until the query has completed, and then retrieves the table
result and processes it in accordance with the supplied parameters.
The query may or may not be deleted from the server as part of the operation.
</p>

&tapresume-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tapresume</code>:
<dl>

<dt><verbatim>
stilts tapresume joburl='http://dc.zah.uni-heidelberg.de/__system__/tap/run/tap/async/d4ENGR'
                 out=result.csv ofmt=csv
</verbatim></dt>
<dd><p>Resumes waiting for the output of a query on a job with ID
    <code>d4ENGR</code> which was previously started on the GAVO TAP server.
    When it has completed the output table will be written as a
    comma-separated value file.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tapskymatch" tocleaf="yes">
<subhead><title><code>tapskymatch</code>:
                &tapskymatch-purpose;</title></subhead>

<p><code>tapskymatch</code> allows you to perform a positional crossmatch
of a local table with one held in a remote TAP service, as long as
that TAP supports upload queries.
This task does three main jobs.
First, it prepares the ADQL queries and TAP negotiations for you
so that you don't need to remember the syntax for performing
positional crossmatches against a TAP service.
Second, it organises data transfer so that only those columns
required (basically the positional ones) are transmitted to and
from the service, to save on bandwidth.
And third it divides the job up into chunks,
so that the TAP service only has to perform a manageable-sized
query at a time.
If the job is large this chunking can be useful to monitor
progress of the job,
and it also allows you to perform a match which would otherwise
hit the upload or output limits imposed by the service.
</p>

<p>The positional match may be done in any spherical coordinate system,
it's up to the user to ensure that the same coordinates are provided
for the local and remote tables.
</p>

<p>Note that <ref id="cdsskymatch"><code>cdsskymatch</code></ref>
provides similar functionality by accessing a different external service,
which is usually much faster;
if the table you wish to match is part of the VizieR database,
you may wish to use that command instead.
</p>

&tapskymatch-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tapskymatch</code>:
<dl>

<dt><verbatim>
stilts tapskymatch tapurl=http://dc.g-vo.org/tap
                   taptable=twomass.data taplon=raj2000 taplat=dej2000
                   in=dr5qso.fits inlon=RA inlat=DEC sr=0.00027 find=all
                   out=qso_2mass.fits
</verbatim></dt>
<dd><p>Matches a local catalogue <code>dr5qso.fits</code> against the
    table named <code>twomass.data</code> in the GAVO TAP service.
    The search radius is 1/3600 degrees (1 arcsecond) and all 2MASS
    sources within the radius of each input source are returned.
    </p>
    <p>If you run the command with "<code>stilts -verbose ...</code>"
    the text of the ADQL query submitted to the TAP service will
    (amongst other things) be logged on the console, and you will also
    see the number of rows uploaded and matched in each chunk.
    </p></dd>

<dt><verbatim>
stilts tapskymatch tapurl=http://dc.g-vo.org/tap
                   taptable=rave.dr3 taplon=raj2000 taplat=dej2000
                   tapcols=name,raj2000,dej2000,pmra,pmde
                   in=hip_main.fits inlon=RAdeg inlat=DEdeg
                   icmd='keepcols "HIP RAdeg DEdeg pmra pmde"'
                   sr=0.00027
                   icmd='select nearMoc(\"III/265/ravedr3\",RAdeg,DEdeg,.00027)'
                   icmd=cache icmd=progress
                   blocksize=5000
                   fixcols=all suffixin=_hip suffixremote=_rave
                   find=best
                   omode=topcat
</verbatim></dt>
<dd><p>This matches a local copy of the Hipparcos survey against
    a remote copy of the RAVE survey with a 1-arcsecond radius.
    The output table contains only the identifier, position and proper
    motion columns from
    both the input table (by using the <code>keepcols</code> filter)
    and the remote table (by specifying <code>tapcols</code>);
    the other columns are discarded.
    The <code>fixcols</code> and <code>suffix</code>* parameters
    ensure that a suffix is added to all the output column names,
    <code>_hip</code> for the input (Hipparcos) columns and
    <code>_rave</code> for the remote (RAVE) ones.
    </p>
    <p>Before uploading, the input table is preprocessed by selecting only
    those rows that fall within the actual footprint of the RAVE survey,
    by filtering with a MOC giving RAVE coverage
    (the RAVE dr3 MOC is also available at
    <webref url="http://alasky.u-strasbg.fr/footprints/tables/vizier/III_265_ravedr3/MOC"
            >this URL</webref>).
    This step reduces the amount of data that needs to be uploaded,
    since only those rows in the given coverage region stand a chance of
    having a match in the remote table.
    Note use of the <code>nearMoc</code> function with the value of the
    match radius as the fourth parameter; this includes those objects
    which may be outside the actual MOC region but close enough that
    a match could still result.
    </p>
    <p>The <code>blocksize</code> parameter determines the number of rows
    uploaded at a time.  If you receive warnings that the output has
    been truncated, you should decrease this number.
    </p>
    <p>Progress is displayed as the match continues.
    The <code>cache</code> filter must be applied upstream of (before)
    the <code>progress</code> filter itself for this to work,
    since otherwise the match processing reads all the input rows 
    before the actual work is done, and the progress monitor completes
    before the match actually starts.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tcat" tocleaf="yes">
<subhead><title><code>tcat</code>: &tcat-purpose;</title></subhead>

<p><code>tcat</code> is a tool for concatenating any number of 
similar tables one after the other.
The tables must be of similar form to each other (same number and
types of columns).  Preprocessing of the tables may be done using
the <code>icmd</code> parameter, which will operate in the same
way on all the input tables.  Table parameters of the output table
will be taken from the first of the input tables.
</p>

<p>Subject to some constraints on the details of the input and output
formats and processing, <code>tcat</code> is capable of joining
an unlimited number of tables together to produce an output table of
unlimited length, without large memory requirements.
If there are very many input files, it may be necessary to set the
<code>lazy</code> parameter so that they are not all kept open
at once.
</p>

<p>If you have heterogeneous tables, in different formats or
requiring different preprocessing steps from each other before
they can be concatenated, use <code><ref id="tcatn">tcatn</ref></code>
instead.
</p>

&tcat-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tcat</code>:
<dl>

<dt><verbatim>
stilts tcat ifmt=ascii in=t1.txt in=t2.txt in=t3.txt out=table.txt
</verbatim></dt>
<dd><p>Concatenates the three named ASCII format tables to produce
    an output table.  All three must have compatible numbers and types 
    of columns.
    </p></dd>

<dt><verbatim>
stilts tcat ifmt=ascii in="t1.txt t2.txt t3.txt" out=table.txt
</verbatim></dt>
<dd><p>Has exactly the same effect as the previous example.
    </p></dd>

<dt><verbatim>
stilts tcat ifmt=ascii in=&indirection.char;inlist.lis out=table.txt
</verbatim></dt>
<dd><p>This will have the same effect as the previous two examples if a
    file name "inlist.lis" in the current directory contains three lines,
    "t1.txt", "t2.txt" and "t3.txt".
    </p></dd>

<dt><verbatim>
stilts tcat in=&indirection.char;infits.lis out=gaia_source.colfits
            lazy=true countrows=true
</verbatim></dt>
<dd><p>Concatenates the contents of all the files listed in "infits.lis"
    to one large output colfits file.
    The <code>lazy=true</code> is a good idea if there is a large number
    of input files listed.
    </p></dd>

<dt><verbatim>
stilts tcat in=r368776.fits#1 in=r368776#2 in=r368776.fits#3 in=r368776.fits#4
            out=r368776_all.fits
</verbatim></dt>
<dd><p>Concatenates the contents of four tables (the first four extension HDUs)
    from a multi-extension FITS file to produce a single FITS table.
    Many Unix shells (csh, bash) will allow you to list the input files
    using the following shorthand: "<code>in=r368776.fits#{1,2,3,4}</code>".
    </p></dd>

<dt><verbatim>
stilts tcat in=r368776.fits multi=true out=r368776_all.fits
</verbatim></dt>
<dd><p>Concatenates all the tables in the named file together.
    Setting <code>multi=true</code> means that instead of picking the first
    table from each named <code>in</code> table, all tables will be selected.
    So, if the input FITS file in this example has just four table HDUs,
    then this example does exactly the same as the previous one,
    but with less typing.
    The same thing works with multi-TABLE VOTable documents, but most other
    file formats (CSV etc) do not have the facility for storing multiple
    tables in a single file.
    </p></dd>

<dt><verbatim>
stilts tcat in=r368776.fits multi=true out=r368776_all.fits
            icmd=progress seqcol=ID
</verbatim></dt>
<dd><p>Does the same as the previous example with a couple of additions.
    Firstly, progress through each of the input files will be reported
    to the console.
    Secondly, an additional column "ID" will be appended to the output which 
    contains 1 for all the rows from the first input table, 2 for the 
    rows from the second one and so on.
    </p></dd>

<dt><verbatim>
stilts tcat in='rA.csv rB.csv rC.csv' ifmt=csv \
            icmd='keepcols "RA DEC FLUX"' icmd='sorthead 10 FLUX' \
            ocmd='sort FLUX'
</verbatim></dt>
<dd><p>Takes the 10 rows with highest FLUX values from each of three input
    tables (in comma-separated value format) and joins them together to
    produce a 30-row output table.  This is then sorted in FLUX order, 
    and the resulting table is output to the console in text format.
    Only the columns RA, DEC and FLUX are output; any other columns 
    are discarded.  The input tables don't need to have identical forms
    to each other, but each must have at least an RA, DEC and FLUX column.
    </p></dd>

<dt><verbatim>
stilts tcat in=vizier.xml multi=true
            icmd='keepcols "&ucd.prefix;RECORD &ucd.prefix;POS_EQ_RA_MAIN &ucd.prefix;POS_EQ_DEC_MAIN"'
            uloccol=TID out=all.csv
</verbatim></dt>
<dd><p>This processes a VOTable file which may have multiple TABLEs in it,
    but for which each of the tables is known to have columns with the
    UCDs RECORD, POS_EQ_RA_MAIN and POS_EQ_DEC_MAIN (this is typical of
    VOTables retrieved from CDS's VizieR service).
    It retains only those columns from each table and writes the result
    as a single concatenated table to a CSV file.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tcatn" tocleaf="yes">
<subhead><title><code>tcatn</code>: &tcatn-purpose;</title></subhead>

<p><code>tcatn</code> is a tool for concatenating a number of tables 
one after the other.  Each table can be manipulated separately 
prior to the concatenatation.
If you have two tables T1 and T2 which contain similar columns, and you 
want to treat them as a single table, you can use <code>tcatn</code>
to produce a new table whose metadata (row headings etc) comes from T1
and whose data consists of all the rows of T1 followed by all the rows of T2.
</p>

<p>For this concatenation to make sense, each column of T1 must be
compatible with the corresponding column of T2 - they must have 
compatible types and, presumably, meanings.
If this is not the case for the tables that you wish to concatenate,
for instance the columns are in different orders, or the units 
differ between a column in T1 and its opposite number in T2,
you can use the <code>icmd1</code> and/or <code>icmd2</code>
parameters to manipulate the input tables so that the
column sequences are compatible.  See <ref id="tcatn-examples"/> for
some examples.
</p>

<p>If the tables are similar to each other 
(same format, same columns, same preprocessing stages required if any),
you may find it easier to use <code><ref id="tcat">tcat</ref></code> instead.
</p>

&tcatn-summary;

<subsubsect id="tcatn-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tcatn</code>:
<dl>

<dt><verbatim>
stilts tcatn nin=2 in1=obs1.fits in2=obs2.fits out=combined.fits
</verbatim></dt>
<dd><p>Concatenates two similar observation catalogues to form a combined one.
    In this case, both input and output tables are FITS files.
    </p></dd>

<dt><verbatim>
stilts tcatn nin=3 omode=stats in1=obs1.txt ifmt1=ascii
                               in2=obs2.xml ifmt2=votable
                               in3=obs3.fit ifmt3=fits
</verbatim></dt>
<dd><p>Three catalogues with similar forms but in different data formats
    are joined.  Instead of writing the result to an output file, 
    the resulting joined catalogue is examined to calculate
    its statistics, which are written to standard output.
    </p></dd>

<dt><verbatim>
stilts tcatn nin=2 in1=survey.vot.gz ifmt2=csv in2=more_data.csv
             icmd1='addskycoords fk5 galactic RA2000 DEC2000 GLON GLAT' \
             icmd1='keepcols "OBJ_ID GLON GLAT"' \
             icmd2='keepcols "ident gal_long gal_lat"' \
             loccol=FILENAME
             omode=topcat
</verbatim></dt>
<dd><p>In this case we are trying to concatenate results from two tables
    which are quite dissimilar to each other.  In the first place,
    one is a VOTable (no <code>ifmt1</code> parameter is required since
    VOTables can be detected automatically), and the other is a 
    comma-separated-values file (for which the <code>ifmt2=csv</code> 
    parameter must be given).
    In the second place, the column structure of the two tables may be
    quite different.  By pre-processing the two tables using the
    <code>icmd1</code> &amp; <code>icmd2</code> parameters, we produce
    in each case an input table which consists of three columns of
    compatible types and meanings: an integer identifier and floating point
    galactic longitude and latitude coordinates.
    The second table contains such columns to start with,
    but the first table requires an initial step to convert 
    FK5 J2000.0 coordinates to galactic ones.
    <code>tcatn</code> joins the two doctored tables together, to produce
    a table which contains only these three columns, with all the rows
    from both input tables, and sends the result directly 
    to a new or running instance of TOPCAT.
    An additional column named FILENAME is appended to the table
    before sending it; this contains "survey.vot.gz" for all the columns
    from the first table and "more_data.csv" for all the columns from
    the second one.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>


<subsect id="tcopy" tocleaf="yes">
<subhead><title><code>tcopy</code>: &tcopy-purpose;</title></subhead>

<p><code>tcopy</code> is a table copying tool.
It simply copies a table from one place to another, but since
you can specify the input and output formats as desired, it works
as a converter from any of the supported 
<ref id="inFormats" plaintextref="yes">input formats</ref> 
to any of the supported 
<ref id="outFormats" plaintextref="yes">output formats</ref>.
</p>

<p><code>tcopy</code> is just a stripped-down version of
<ref id="tpipe"><code>tpipe</code></ref> - it doesn't do anything
that <code>tpipe</code> can't, but the usage is slightly 
simplified.
It is provided as a drop-in replacement for the old
<code>tablecopy</code> (<code>uk.ac.starlink.table.TableCopy</code>)
tool which was supplied with earlier versions of STIL and TOPCAT - 
it has the same arguments and behaviour as <code>tablecopy</code>, 
but is implemented somewhat differently
and will in some cases be more efficient.
</p>

&tcopy-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tcopy</code> in use:
<dl>

<dt><verbatim>
stilts tcopy stars.fits stars.xml
</verbatim></dt>
<dd><p>Copies a FITS table to a VOTable.
    Since no input format is specified, the format is automatically 
    detected (FITS is one of the formats for which this is possible).
    Since no output format is specified, the <code>stars.xml</code>
    filename is examined to make a guess at the kind of output to write:
    the <code>.xml</code> ending is taken to mean a TABLEDATA-encoded
    VOTable.
    </p></dd>

<dt><verbatim>
stilts tcopy stars.fits stars.xml ifmt=fits ofmt=votable
</verbatim></dt>
<dd><p>Does the same as the previous example, but the input and output
    formats have been specified explicitly.
    </p></dd>

<dt><verbatim>
stilts tcopy ofmt=text http://remote.host/data/vizer.xml.gz#4 -
</verbatim></dt>
<dd><p>Prints the contents of a remote, compressed VOTable to the terminal in 
    a human-readable form.  The <code>#4</code> at the end of the URL
    indicates that the data from the fifth <code>TABLE</code> element
    in the remote document are to be used.  The gzip compression of
    the table is taken care of automatically.
    </p></dd>

<dt><verbatim>
stilts tcopy ifmt=csv ofmt=latex spec.csv
</verbatim></dt>
<dd><p>Converts a comma-separated values file to a LaTeX table environment,
    writing the result to standard output.
    </p></dd>

<dt><verbatim>
stilts -classpath /usr/local/jars/pg73jdbc3.jar \
       -Djdbc.drivers=org.postgresql.Driver \
       tcopy in="jdbc:postgresql://localhost/imsim#SELECT ra, dec, Imag FROM dqc" \
             ofmt=fits wfslist.cat
</verbatim></dt>
<dd><p>Makes an SQL query on a PostgreSQL database and writes the
    results to a FITS file.
    The whole command is shown here, to show that the 
    classpath is augmented to include the PostgreSQL
    driver class, and the driver class is named using the 
    <code>jdbc.drivers</code> system property.
    As you can see, using SQL from Java is a bit fiddly, 
    and there are other ways to perform this
    setup than on the command line - see <ref id="jdbcConfig"/>
    and <code><ref id="tpipe">tpipe</ref></code>'s 
    <code>omode=tosql</code> output mode.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tcube" tocleaf="yes">
<subhead><title><code>tcube</code>: &tcube-purpose;</title></subhead>

<p><code>tcube</code> constructs an N-dimensional histogram, or density map, 
from N columns of an input table, and writes it out as an
N-dimensional data cube.  The parameters you supply define which N
numeric columns of the input table you want to use and the dimensions 
(bounds and pixel sizes) of the output grid,
as well as any weighting to be applied to each point and how
the weighted quantities in a single bin are to be aggregated together.
Each table row then defines a point in N-dimensional space.  
The program goes through each row, and if the point that row
defines falls within the bounds of the output grid you have defined,
associates the weight value with the relevant pixel in the output grid.
When all the input values have been processed, the weights in each
pixel are aggregated according to the requested combination method.
</p>

<p>The resulting N-dimensional array, whose pixel values represent an
aggregation of the rows associated with that region of the N-dimensional space,
is then written out as a FITS file.
In one dimension, this gives you a normal histogram of a given variable.
In two dimensions it might typically be used to plot the density or
weighted density on the sky of objects from a catalogue.
</p>

<p>As with some of the other generic table commands,
you can perform extensive pre-processing on the input table by
use of the <code>icmd</code> parameter before the actual cube 
counts are calculated.
</p>

<p>See also <ref id="tgridmap"><code>tgridmap</code></ref>,
which does a similar job to this command but writes the output
in table format.
</p>

&tcube-summary;

<subsubsect id="tcubeExamples">
<subhead><title>Examples</title></subhead>

<p>
<dl>

<dt><verbatim>
stilts tcube in=2QZ_6QZ_pubcat.fits out=ccm.fits \
             cols='Bj_R U_Bj Bj' binsizes='0.05 0.05 0.5' bounds='-2:1 -3:2 :'
</verbatim></dt>
<dd><p>Calculates a 3-dimensional colour-colour-magnitude grid from 
    three existing columns in a table.  The bin (pixel) sizes are specified.
    The data bounds are specified explicitly for the (first two) 
    colour dimensions, but for the (third) magnitude dimension it is 
    determined from the minimum and maximum values the data in 
    that column of the table.
    The output is a three-dimensional FITS cube containing the number
    of points in each cell.
    </p></dd>

<dt><verbatim>
stilts tcube in=iras_psc.vot out=flux60_map.fits \
             icmd='addskycoords fk5 galactic ra dec glat glon' \
             cols='glat glon' nbins='400 200' \
             scale=Fnu_60 combine=sum 
</verbatim></dt>
<dd><p>Calculates a map of integrated fluxes in galactic coordinates from
    a catalogue of IRAS point sources.  The output is a two-dimensional
    FITS image representing the sky in galactic coordinates,
    with each pixel containing the sum of <code>Fnu_60</code> fluxes
    from objects in that rectangle of <code>(glat,glon></code> space.
    Bounds are determined automatically from the data, and the number of
    pixels in each dimension (400 in latitude and 200 in longitude) are
    specified, which means that the pixel sizes don't have to be.
    Since the input table contains sky positions in equatorial 
    coordinates rather than galactic ones, the <code>addskycoords</code>
    filter is used to preprocess the data before the cube generation
    step (see <ref id="filterSteps"/>).
    </p></dd>

</dl>
</p>

</subsubsect>
</subsect>

<subsect id="tloop" tocleaf="yes">
<subhead><title><code>tloop</code>: &tloop-purpose;</title></subhead>

<p><code>tloop</code> generates a one-column table where the values in
the column are effectively populated from a for loop
(start, end, step).
This may be useful as it is, or it can be postprocessed with
<code>ocmd</code> parameters to add more columns etc.
</p>

&tloop-summary;

<subsubsect id="tloopExamples">
<subhead><title>Examples</title></subhead>

<p>
<dl>

<dt><verbatim>
stilts tloop COUNTER 0 1000
</verbatim></dt>
<dd><p>Generates a table with a single column, named <code>COUNTER</code>,
    and a thousand rows.
    The value in the first row is 0 and in the last row is 999.
    The table is written to standard output.
    </p></dd>

<dt><verbatim>
stilts tloop time 0 10 0.25 out=times.csv
</verbatim></dt>
<dd><p>Generates a table with one column <code>time</code> counting from 
    0 to 9.75 in steps of 0.25.  Output is to a CSV file.
    The parameters here are specified in order, but could equivalently
    be given by name:
    "<code>stilts tloop var=time start=0 end=10 step=0.26</code>".
    </p></dd>

<dt><verbatim>
stilts tloop x start=1 end=11 ocmd='addcol x2 x*x' ocmd='addcol x3 x*x*x'
             ocmd='stats name sum'
</verbatim></dt>
<dd><p>Generates a table with a column <code>x</code> running from 1 to 10
    inclusive.  The <code>addcol</code> filters then append two further
    columns, giving the squares and cubes of these values respectively,
    giving a table of 10 rows and 3 columns.
    Finally this table is piped through a <code>stats</code> filter to
    calculate the sums of the values, squares and cubes in this range.
    </p></dd>
</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tgridmap" tocleaf="yes">
<subhead><title><code>tgridmap</code>: &tgridmap-purpose;</title></subhead>

<p><code>tgridmap</code> scans an input table to create one or more
N-dimensional density maps, or equivalently N-dimensional histograms,
of the values in an input table, and outputs the result as an,
optionally sparse, table containing a row for each grid cell.
The maps/histograms can optionally be weighted by some quantity from the
input table, and various options such as summing, averaging and counting
are available for aggregation of inputs into the output bins.
</p>

<p>The supplied <code>coords</code> parameter defines
which N numeric columns of the input table form the coordinates
of the bin grid, and the <code>cols</code> parameter defines which
quantities are aggregated into each bin.
Either the <code>binsizes</code> or <code>nbins</code> parameter must
be supplied to define the extents of the bins on each axis.
The output table contains a row for each bin,
with columns giving the central (and upper/lower bound) values of each
grid coordinate, and a column for each aggregated value.
The rows are output in first-coordinate-slowest sequence,
and the <code>sparse</code> parameter determines whether a row is
written for every cell in the hypercube defined by the grid dimensions,
or only for those cells with non-blank data.
</p>

<p>The tabular form of the output may not be the most appropriate or
compact way to write a density map, especially for multi-dimensional grids,
but it means the output can be manipulated later by other STILTS commands
or by TOPCAT.
To do a similar job with more compact output,
see <ref id="tcube"><code>tcube</code></ref>.
See also <ref id="tskymap"><code>tskymap</code></ref>,
which does the same thing for sky geometry (and is probably a better
choice if you find yourself accumulating onto a longitude-latitude grid).
</p>

&tgridmap-summary;

<subsubsect id="tgridmapExamples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tgridmap</code>:

<dl>

<dt><verbatim>
stilts tgridmap in=ravedr4.fits coords=HRV nbins=20
</verbatim></dt>
<dd><p>Calculates a simple 1-dimensional unweighted histogram of the
    HRV column from the input table <code>ravedr4.fits</code>.
    The output is a table with columns
    <code>HRV</code>, giving the central value of each bin, and
    <code>COUNT</code>, giving the number of input rows with HRV
    values in that bin;
    additional columns <code>HRV_lo</code> and <code>HRV_hi</code> give the
    lower and upper bounds of the bin.
    The bin size is determined from the actual range of the HRV values
    in the input table, combined with the requested bin count of 20;
    however, the bin size will be chosen as some round number,
    so the bin count (number of rows in the output table)
    may not be exactly as requested.
    </p></dd>

<dt><verbatim>
stilts tgridmap in=ravedr4.fits coords=HRV
                binsizes=100 bounds=-450:450 sparse=false
</verbatim></dt>
<dd><p>Produces a similar histogram to the previous example,
    but the bin dimensions, ranges and alignments are specified explicitly
    rather than being worked out from the data.
    There will be 9 output bins, [-450,-350), [-350,-250), ..., [350,450);
    any values outside of these bins will be ignored.
    The <code>sparse=false</code> parameter means that rows will be output
    for all 9 bins, even if some of them are empty.
    Note supplying bin geometry in this way allows control of bin boundaries;
    in this case HRV=0 is in the middle of a bin not at a bin boundary.
    This will also be faster, since no initial scan to determine
    actual data ranges has to be performed.
    </p></dd>

<dt><verbatim>
stilts tgridmap in=edr3-local.fits
                icmd='addcol nobs astrometric_n_good_obs_al'
                icmd='addcol g_abs phot_g_mean_mag+5*log10(parallax*0.01)'
                coords='bp_rp g_abs'
                binsizes='0.125 0.5' bounds='-1:6 -5:20'
                cols='1;count;NUM nobs;sum;SUM_NOBS nobs;mean;MEAN_NOBS'
                out=grid-stats.vot sparse=false
</verbatim></dt>
<dd><p>This assembles a table containing three weighted histograms on a
    2-d colour vs. absolute-magnitude grid.
    The output table contains columns giving
    <code>bp_rp</code> and <code>g_abs</code> coordinate values
    for each grid point,
    as well as columns NUM containing source density,
    and columns SUM_NOBS and MEAN_NOBS containing respectively
    the sum and mean of the <code>nobs</code> column in each grid cell.
    Since <code>sparse=false</code> the number and arrangement
    of output rows is determined by the binsizes and bounds (57*51 rows)
    independent of the input data, and could be compared
    with similar runs on different input tables.
    The <code>icmd=addcol...</code> parameters prepare values for accumulation
    ahead of the actual gridding step for convenience though this isn't
    essential, the relevant expressions could be used directly in the
    <code>coords</code> and <code>cols</code> parameters if preferred.
    </p></dd>

</dl>
</p>
</subsubsect>

</subsect>

<subsect id="tgroup" tocleaf="yes">
<subhead><title><code>tgroup</code>: &tgroup-purpose;</title></subhead>

<p><code>tgroup</code> identifies groups of rows in
a table based on the values in a given column or columns,
and calculates statistical quantities or otherwise collapses
down the multiple values from other columns into single values
representing each group.
It does the same job as a <code>SELECT ... GROUP BY</code> statement
with aggregate functions in ADQL/SQL.
</p>

<p>The <code>keys</code> parameter defines how input rows are grouped,
and the <code>aggcols</code> parameter defines what quantities to
aggregate from the rows in each group.
<code>keys</code> specifies one or more values (column names or expresssions)
that must be the same for rows grouped together,
while <code>aggcols</code> specifies zero or more columns to be
added based on the content of rows in each group.
The output table therefore contains
one column for each entry in <code>keys</code>
and one column for each entry in <code>aggcols</code>,
and has one row for each group identified.
</p>

<p>This command can therefore be used to count rows or calculate
statistical quantities per group.
A number of statistical aggregation methods are provided such as
mean, median, minimum, maximum etc.
For more specialised requirements, for instance quantiles or
custom statistics, you can also use the <em>array</em>
aggregators which generate an array containing all of the
values in the group, and operate on the resulting column using
one of the functions in the <ref id="Arrays">Arrays</ref> class.
</p>

<p>By way of comparison, the <code>tgroup</code> invocation:
<verbatim>
   stilts tgroup in=t
                 keys="year detector"
                 aggcols="0;count;num gmag;min;min_gmag gmag;mean"
</verbatim>
corresponds roughly to the ADQL query:
<verbatim>
   SELECT COUNT(*) AS num, MIN(gmag) AS min_gmag, MEAN(gmag),
   FROM t
   GROUP BY year, detector
</verbatim>
</p>

<p>See also the
<ref id="tgridmap"><code>tgridmap</code></ref> and
<ref id="tskymap"><code>tskymap</code></ref> commands,
which provide similar functionality where the grouping is over
evenly spaced numeric/coordinate values.
</p>

&tgroup-summary;

<subsubsect id="tgroupExamples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tgroup</code>:
<dl>

<dt><verbatim>
stilts tgroup in=B_sn.vot keys='MType' aggcols='null;count' out=type_counts.csv
</verbatim></dt>
<dd><p>This produces an output table with two columns:
    the first column (defined by the <code>keys</code> parameter)
    gives all the distinct values of the
    <code>MType</code> column in the input table,
    and the second column (defined by the <code>aggcols</code> parameter)
    gives a count of how many rows in the input table have that MType value.
    The output is written to a CSV file.
    </p>
    <p>Since the <code>count</code> aggregator pays no attention to the
    values it is counting, the quantity before the semicolon is
    irrelevant in this case, so for instance "<code>0;count</code>",
    "<code>100;count</code>" or "<code>MType;count</code>"
    would work just as well.
    If the <code>aggcols</code> parameter is omitted altogether,
    the output table will be the same but without the
    <code>count</code> column,
    i.e. it will just list all the distinct values of
    the <code>MType</code> column.
    </p></dd>

<dt><verbatim>
stilts tgroup in=B_sn.vot
              icmd='colmeta -name Discoverer disc' icmd='select MaxMag&lt;20'
              keys='Discoverer' aggcols='0;count;SNae'
              ocmd='sorthead -down 10 SNae'
</verbatim></dt>
<dd><p>This does a similar job to the previous example,
    but with some additional pre- and post-processing,
    to produce a league table of discoverers of bright supernovae.
    The <code>icmd</code> parameters define pre-processing filters
    that rename one column, and select only those rows for which
    the <code>MaxMag</code> column is below a certain threshold.
    The <code>keys</code> parameter groups rows by the contents of the
    <code>Discoverer</code> (n&#xe9;e <code>disc</code>) column,
    and the <code>aggcols</code> parameter counts how many sources
    are listed for each discoverer,
    naming the resulting column "<code>SNae</code>".
    Finally, the <code>ocmd</code> filter
    sorts the results by the count column,
    listing only the top ten to standard output.
    </p></dd>

<dt><verbatim>
stilts tgroup in=exomercat.fits
              keys='discovery_method status'
              aggcols='null;count mass;mean mass;stdev r;max;max_radius r;ngood;num_radii'
              ocmd='sort "discovery_method status"'
</verbatim></dt>
<dd><p>In this case there are two columns listed in
    the <code>keys</code> parameter,
    which means each output row groups all the input rows with the same
    <code>discovery_method</code> <em>and</em> <code>status</code> values.
    For each such group several values are calculated and output
    in separate columns: number of entries, mean mass,
    standard deviation of mass, maximum radius,
    and number of entries with a non-blank radius.
    </p></dd>

<dt><verbatim>
stilts tgroup in=dr16qso.fits
              icmd='addcol -units yr year (int)mjdToDecYear(mjd)'
              keys='year'
              aggcols='z;ngood;nz'
              aggcols='z;array;z_values'
              ocmd='addcol z_q1 quantile(z_values,0.25)'
              ocmd='addcol z_q2 quantile(z_values,0.50)'
              ocmd='addcol z_q3 quantile(z_values,0.75)'
              ocmd='delcols z_values'
              out=zq-by-year.fits
</verbatim></dt>
<dd><p>This groups the rows in an input table by calendar year,
    and for each year calculates the number of non-blank items,
    and the quartiles, of the values in the <code>z</code> column.
    Since no quartile aggregator is provided,
    aggregation is first done to an array of all the z values per year,
    and quartiles are calculated using the
    <ref id="Arrays-quantile">quantile</ref> function, which takes an
    array value as well as the quantile point required.
    Before writing the output, the bulky array column is removed.
    </p>
    <p>In this example, the <code>aggcols</code> parameter has been
    given three times for the three aggregated values.
    This is just another way to specify multiple entries, and it could
    equally have been written in one go with a space as a delimiter, i.e.
    "<code>aggcols='z;ngood;nz z;array;z_values'</code>".
    </p>
    <p>Note that since the grouped quantity here is numeric and evenly spaced,
    this job could equally have been done using the
    <ref id="tgridmap"><code>tgridmap</code></ref> command, like
    <verbatim>
    stilts tgridmap in=dr16qso.fits
                    icmd='addcol -units yr year mjdToDecYear(mjd)'
                    coords=year binsizes=1
                    cols='1;count;nz z;Q1;z_q1 z;median;z_q2 z;Q3;z_q3'
    </verbatim>
    </p></dd>
</dl>
</p>
</subsubsect>

</subsect>

<subsect id="tjoin" tocleaf="yes">
<subhead><title><code>tjoin</code>: &tjoin-purpose;</title></subhead>

<p><code>tjoin</code> performs a trivial side-by-side join of multiple tables.
The N'th row of the output table consists of the N'th row of the
first input table, followed by the N'th row of the second input table, ...
and so on.  It is suitable if you want to amalgamate two or more 
tables whose row orderings correspond exactly to each other.
</p>

<p>For the (more usual) case in which the rows of the tables to be 
joined are not already in the right order, use one of the
<ref id="match" plaintextref="yes">crossmatching commands</ref>.
</p>

&tjoin-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tjoin</code>:
<dl>

<dt><verbatim>
stilts tjoin nin=2 in1=positions.fit in2=fluxes.fits out=combined.fits
</verbatim></dt>
<dd><p>Takes two input FITS files and sticks them together side by side,
    writing the result as a third FITS file.  The output will have the
    same number of rows as each of the input catalogues, and a number
    of columns equal to the sum of those in the two input catalogues.
</p></dd>

<dt><verbatim>
stilts tjoin nin=3 fixcols=all \
             ifmt1=ascii in1=t1.txt suffix1=_T1 \
             ifmt2=ascii in2=t2.txt suffix2=_T2 \
             ifmt3=ascii in3=t3.txt suffix3=_T3 \
             ocmd='select FLAG_T1==0' \
             omode=stats
</verbatim></dt>
<dd><p>This joins three ascii tables together.  Each column of the output
    table is renamed by appending a string to it ("_T1" for the first 
    table, "_T2" for the second...).
    Only those rows of the output for which the FLAG column in the first
    input table, and hence the FLAG_T1 column in the output table,
    have the value zero are selected.  Statistics are calculated for
    all the columns of these selected rows, and written to the output.
</p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tmatch1" tocleaf="yes">
<subhead><title><code>tmatch1</code>: &tmatch1-purpose;</title></subhead>

<p><code>tmatch1</code> performs efficient and flexible 
crossmatching between the rows of a single table.
It can match rows on the basis of their relative position in the sky,
or alternatively using many other criteria such as separation in 
in some isotropic or anisotropic Cartesian space,
identity of a key value, or some combination of these;
the full range of match criteria is dicussed in <ref id="MatchEngine"/>.
</p>

<p>The basic task performed by the intra-table matcher is to identify
groups of rows within the table which match each other.
See <ref id="matchGroup"/> for an explanation of exactly what 
consitutes a match group.
The result of identifying these groups is expressed as an output table 
in one of a variety of ways, specified by the <code>action</code> parameter.
These options include marking group membership in added columns 
and eliminating some or all rows which form part of a match group.
</p>

&tmatch1-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tmatch1</code>:
<dl>

<dt><verbatim>
stilts tmatch1 matcher=sky values="RA2000 DE2000" params=20 \
               action=keep0 in=crowded.vot out=sparse.vot
</verbatim></dt>
<dd><p>Copies an input catalogue "crowded.vot" to an output catalogue
    "sparse.vot", but omitting any objects (rows) which are within 20 arcsec
    of other objects.  The output catalogue will contain no near neighbours.
    </p></dd>

<dt><verbatim>
stilts tmatch1 matcher=skyerr values="RA2000 DE2000 RADIUS*4" params=40 \
               action=keep0 in=crowded.vot out=sparse.vot
</verbatim></dt>
<dd><p>This is similar to the previous example, but uses the 
    <code>skyerr</code> matcher which determines the proximity threshold
    on a row-by-row basis from values in the table - in this case 4 times
    the value of the RADIUS column (this value must be in arc seconds). 
    The <code>params=40</code> value does not affect the result, but it
    gives the algorithm an idea of the rough scale of object separation.
    </p></dd>

<dt><verbatim>
stilts tmatch1 matcher=3d values="XPIX YPIX ZPIX" params=10 action=identify \
               in=state.fit ocmd='select GroupSize&gt;3' out=groups3+.fit
</verbatim></dt>
<dd><p>Uses the "3d" matcher to identify groups of objects
    in terms of their proximity in a 3-dimensional Cartesian space,
    with positions given by the XPIX, YPIX and ZPIX columns in the input table.
    The <code>action=identify</code> parameter means that the input table
    is written out with the same rows, but with additional columns indicating
    which rows are associated with each other.  One of these columns,
    "GroupSize" gives the number of objects in each group.  
    The postprocessing filter <code>ocmd='select GroupSize&gt;3'</code>
    selects only those rows which are part of groups of three objects or
    larger; singletons and pairs are discarded before writing the output file.
    </p></dd>

<dt><verbatim>
stilts tmatch1 matcher=sky values="ra dec" params=3 action=wide2 \
               ocmd='keepcols "id_1 ra_1 dec_1 id_2 ra_2 dec_2"'
               in=galaxy.fits out=binaries.txt ofmt=ascii
</verbatim></dt>
<dd><p>Identifies pairs of objects within 3 arcsec of each other from an
    input catalogue.  Singles, and groups of three or more, will be 
    discarded.  The output table generated is a double-width version of
    the input table with pairs of objects next to each other on the same row.
    Here, the <code>ocmd</code> post-processing filter discards all of the
    columns except the identifiers and sky positions for each object.
    The output is to a text file.
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tmatch2" tocleaf="yes">
<subhead><title><code>tmatch2</code>: &tmatch2-purpose;</title></subhead>

<p><code>tmatch2</code> is an efficient and highly configurable 
tool for crossmatching pairs of tables.
It can match rows between tables on the basis of their relative position
in the sky, or alternatively using many other criteria such as 
separation in some isotropic or anisotropic Cartesian space, 
identity of a key value, or some combination of these;
the full range of match criteria is discussed in <ref id="MatchEngine"/>.
You can choose whether you want to identify all the matches or
only the closest,
and what form the output table takes, for instance matched rows only,
or all rows from one or both tables, or only the unmatched rows.
</p>

<p>If you simply want to match two tables based on sky position with
a fixed maximum separation, you may find the
<ref id="tskymatch2"><code>tskymatch2</code></ref> command easier to use.
</p>

<p><strong>Note:</strong> the <code>duptag1</code> and <code>duptag2</code>
parameters have been replaced at version 1.4 by 
<code>suffix1</code> and <code>suffix2</code>
for consistency with other table join tasks.
</p>

&tmatch2-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tmatch2</code>:
<dl>

<dt><verbatim>
stilts tmatch2 in1=obs_v.xml in2=obs_i.xml out=obs_iv.xml \
               matcher=sky values1="ra dec" values2="ra dec" params="2"
</verbatim></dt>
<dd><p>Takes two input catalogues (VOTables), one with observations in
    the V band and the other in the I band, and performs a match
    to find objects within 2 arcseconds of each other.
    The result is a new table containing only rows where a match was found.
    </p></dd>

<dt><verbatim>
stilts tmatch2 survey.fits ifmt2=csv mycat.csv \
               icmd1='addskycoords fk4 fk5 RA1950 DEC1950 RA2000 DEC2000' \
               matcher=skyerr \
               params=10 values1="RA2000 DEC2000 POS_ERR"  values2="RA DEC 0" \
               join=2not1 omode=count
</verbatim></dt>
<dd><p>Here a comma-separated-values file is being compared with a FITS
    catalogue representing some survey results.
    Positions in the survey catalogue use the FK4 B1950.0 system,
    and so a preprocessing step is inserted to create new position columns 
    in the first input table using the FK5 J2000.0 system,
    which is what the other input table uses.
    The survey catalogue contains a POS_ERR column which gives the positional
    uncertainty of its entries, so the <code>skyerr</code> matcher is
    used, which takes account of this; the third entry in the 
    <code>values1</code> parameter is the POS_ERR column (in arcsec).
    Since the second input table has no positional uncertainty information,
    0 is used as the third entry in <code>values2</code>.
    The <code>params</code> gives a rough idea of the scale of the object
    separations, but its value does not affect the result.
    The join type is <code>2not1</code>, which means the output table
    will only contain those entries which are in the second input table
    but not in the first one.
    The output table is not stored, but the number of rows it contains
    (the number of objects represented in the CSV file but not the survey)
    is written to the screen.
    </p></dd>

<dt><verbatim>
stilts tmatch2 ifmt1=ascii ifmt2=ascii in1=cat-a.txt in2=cat-b.txt \
               matcher=2d values1='X Y' values2='X Y' params=5 join=1and2 \
               suffix1=_a suffix2=_b \
               ocmd='addcol XDIFF X_a-X_b; addcol YDIFF Y_a-Y_b' \
               ocmd'keepcols "XDIFF YDIFF"' omode=stats
</verbatim></dt>
<dd><p>Two ASCII-format catalogues are matched, where rows are
    considered to match if their X,Y positions are within 5 units of
    each other in some Cartesian space.
    The result of the matching operation is a table of all the matched rows,
    containing columns named X_a, Y_a, X_b and Y_b (along with any others
    in the input tables) - the <code>suffix*</code> parameters describe 
    how the input X and Y columns are to be renamed to avoid duplicate
    column names in the output table.
    To this result are added two new columns, 
    representing the X and Y positional
    difference between the rows from one input table and those from the other.
    The <code>keepcols</code> filter then throws all the other columns away,
    retaining only these difference columns.
    The final two-column table is not stored anywhere, 
    but (<code>omode=stats</code>) 
    statistics including mean and standard deviation 
    are calculated on its columns and displayed to the screen.
    Having done all this, you can examine the average X and Y differences
    between the two input tables for matched rows, and if they differ
    significantly from zero, you can conclude that there is a systematic
    error between the positions in the two input files.
    </p></dd>

<dt><verbatim>
stilts tmatch2 in1=mgc.fits in2=6dfgs.xml join=1and2 find=all \
               matcher=sky+1d params='3 0.5' \
               values1='ra dec bmag' values2='RA2000 DEC2000 B_MAG" \
               out=pairs.fits
</verbatim></dt>
<dd><p>This performs a match with a matcher that combines <code>sky</code>
    and <code>1d</code> match criteria.  This means that the only
    rows which match are those which are
    <em>both</em> within 3 arcsec of each other on the sky
    <em>and</em> and within 0.5 blue magnitudes.
    Note that for both the <code>params</code> and the 
    <code>values1</code> and <code>values2</code> parameters,
    the items for the <code>sky</code> matcher (RA and DEC) 
    are listed first,
    followed by those for the <code>1d</code> matcher (in this case,
    blue magnitude).
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tmatchn" tocleaf="yes">
<subhead><title><code>tmatchn</code>: &tmatchn-purpose;</title></subhead>

<p><code>tmatchn</code> performs efficient and flexible crossmatching
between multiple tables.  
It can match rows on the basis of their relative position in the sky,
or alternatively using many other criteria such as separation in
in some isotropic or anisotropic Cartesian space,
identity of a key value, or some combination of these;
the full range of match criteria is dicussed in <ref id="MatchEngine"/>.
</p>

<p>Since the match criteria define what counts as a match between two
objects, it is not immediately obvious what is meant by a multi-table
match.  In fact the command can work in one of two distinct modes,
controlled by the <code>multimode</code> parameter.
In <code>pairs</code> mode, one table (by default the first input table)
is designated the reference table, and pair matches between each of the
other tables and that one are identified.
In <code>group</code> mode groups of objects from all the input tables are
identified, as discussed in <ref id="matchGroup"/>.
Currently, in both cases an output matched row cannot contain more
than one object from each input table.
Options for output of multiple rows per input table per match may be 
forthcoming in future releases if there is demand.
</p>

<p><code>tmatchn</code> is intended for use with more than two input tables -
see <ref id="tmatch1"><code>tmatch1</code></ref>
and <ref id="tmatch2"><code>tmatch2</code></ref>
for 1- and 2-table crossmatching respectively.
</p>

&tmatchn-summary;

<subsubsect id="tmatchn-examples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tmatchn</code>:
<dl>

<dt><verbatim>
stilts tmatchn multimode=pairs nin=4 matcher=sky params=5 \
       in1=transients.txt ifmt1=ascii values1='alpha delta' \
       in2=2mass_virgo.fits values2='ra2000 dec2000' \
       in3=sdss_virgo.fits values3='ra dec' \
       in4=first_virgo.fits values4='pos_eq_ra pos_eq_dec' \
       out=matches.xml ofmt=votable-binary
</verbatim></dt>
<dd><p>Compares a text-format table "transients.txt" against each of 
    three other catalogues covering the same region of sky, and outputs
    a table which contains a row for each row of "transients.txt" which
    matches (is within 5 arcsec) of an object in any of the other tables.
    </p></dd>

<dt><verbatim>
stilts tmatchn multimode=pairs nin=4 matcher=sky params=5 \
       in1=transients.txt ifmt1=ascii suffix1='_t' values1='alpha delta' \
       in2=2mass_virgo.fits suffix2='_2mass' values2='ra2000 dec2000' \
       in3=sdss_virgo.fits  suffix3='_sdss'  values3='ra dec' \
       in4=first_virgo.fits suffix4='_first' values4='pos_eq_ra pos_eq_dec' \
       fixcols=all join1=always \
       ocmd='keepcols "*_t designation_2mass SDSSName_sdss id_field_first"' \
       out=matches.xml ofmt=votable-binary
</verbatim></dt>
<dd><p>Similar to the previous example but with some doctoring of what the
    output table will look like.  The <code>fixcols=all</code> and 
    <code>suffixN</code> assignments mean that all the columns from the
    input tables will be renamed for output by adding the given suffixes.
    The <code>keepcols</code> filter applied to the output table throws out
    all the columns except the ones from the reference table (<code>*_t</code>)
    and one column from each of the other table giving object identifiers.
    This output table will probably be easier to read (though contain less
    information) than that from the previous example).
    Additionally, the <code>join1=always</code> assignment means that the output
    table will have one row for each row of the reference table 
    (transients.txt), even if no matches are found for it.
    </p></dd>

<dt><verbatim>
stilts tmatchn multimode=group nin=3 matcher=skyerr params=8 \
       in1=Hband.fits values1='RA DEC SEEING*2' \
       in2=Jband.fits values2='RA DEC SEEING*2' \
       in3=Kband.fits values3='RA DEC SEEING*2' \
       omode=topcat
</verbatim></dt>
<dd><p>Performs a group-mode match.  There is no reference table, so that
    an output row will result for each object which is represented in any
    two of the input catalogues. 
    The match takes account of per-object errors equivalent to twice the
    recorded seeing, which is in the region of 8 arcsec.
    Note that this may not operate as expected if the catalogues contain
    multiple distinct objects too close (in comparison to the declared
    separation) to each other.
    The resulting matched table is sent directly to 
    <webref url="http://www.starlink.ac.uk/topcat/">TOPCAT</webref> 
    (if available).
    </p></dd>

</dl>
</p>

</subsubsect>

</subsect>

<subsect id="tmulti" tocleaf="yes">
<subhead><title><code>tmulti</code>: &tmulti-purpose;</title></subhead>

<p><code>tmulti</code> takes multiple input tables and writes them as
separate tables to a single output container file.
The container file must be of some format which can contain more than
one table, for instance a FITS file (which can contain multiple extensions)
or a VOTable document (which can contain multiple TABLE elements).
Filtering may be performed on the tables prior to writing them.
It is not necessary that all the tables are similar 
(e.g. that they all have the same type and number of columns),
but the same processing commands will be applied to all of them.
For more individual control, use the 
<ref id="tmultin"><code>tmultin</code></ref> task.
</p>

&tmulti-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tmulti</code>:
<dl>

<dt><verbatim>
stilts tmulti ifmt=ascii in=t1.txt in=t2.txt in=t3.txt
              ofmt=fits out=tables.fits
</verbatim></dt>
<dd><p>Takes the three named ASCII format tables and writes them into a
    multi-extension FITS file, as three separate BINTABLE HDUs.
    These tables do not need to be of the same shape or otherwise similar.
    </p></dd>

<dt><verbatim>
stilts tmulti ifmt=ascii in="t1.txt t2.txt t3.txt"
              ofmt=fits out=tables.fits
</verbatim></dt>
<dd><p>Does exactly the same as the previous example.
    </p></dd>

<dt><verbatim>
stilts tmulti ifmt=ascii in=&indirection.char;inlist.lis
              ofmt=fits out=tables.fits
</verbatim></dt>
<dd><p>This will have the same effect as the previous two examples if a
    file name "inlist.lis" in the current directory contains three lines,
    "t1.txt", "t2.txt" and "t3.txt".
    </p></dd>

<dt><verbatim>
stilts tmulti in=extract.fits multi=true out=extract.vot
</verbatim></dt>
<dd><p>This takes the table extensions from a multi-extension FITS file
    and writes them out as a multi-TABLE VOTable document.
    The <code>multi=true</code> setting is required, since this means
    that all the tables from the input file are used as input;
    if it was set false, only the first table HDU from the input file
    would be used.
    </p></dd>

<dt><verbatim>
stilts tmulti in=extract.fits multi=true out=extract.vot
              icmd='badval -999 *MAG'
</verbatim></dt>
<dd><p>Does the same as the previous example, but additionally replaces
    with a blank value occurrences of the value "-999" in columns
    whose name ends with "MAG" in any of the input tables before
    copying them.
    </p></dd>

</dl>
</p>
</subsubsect>

</subsect>

<subsect id="tmultin" tocleaf="yes">
<subhead><title><code>tmultin</code>: &tmultin-purpose;</title></subhead>

<p><code>tmultin</code> takes multiple input tables and writes them
to a single output container file.  
The container file must be of some format which can contain more than
one table, for instance a FITS file (which can contain multiple extensions)
or a VOTable document (which can contain multiple TABLE elements).
Individual filtering may be performed on the tables prior to writing them,
and their formats may be specified individually.
If you want to apply the same pre-processing to all the input tables,
you may find the 
<ref id="tmulti"><code>tmulti</code></ref> command more convenient.
</p>

&tmultin-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tmultin</code>:
<dl>

<dt><verbatim>
stilts tmultin nin=3 in1=t1.xml ifmt1=votable
                     in2=t2.fit ifmt2=fits
                     in3=t3.txt ifmt3=ascii
                     out=tables.fits
</verbatim></dt>
<dd><p>Takes three input tables in different formats, and
writes them out as a single multi-extension FITS file.
</p></dd>

<dt><verbatim>
stilts tmultin nin=3 in1=data.fits icmd1='every 10;   head 100'
                     in2=data.fits icmd2='every 100;  head 100'
                     in3=data.fits icmd3='every 1000; head 100'
                     out=samples.xml ofmt=votable
</verbatim></dt>
<dd><p>Writes three hundred-row tables as separate TABLE elements in
a single output VOTable document.
Each of the output tables is a sample from the same input table,
but sampled differently; the first is every tenth row, the second
every hundredth, and the third every thousandth.
</p></dd>

</dl>
</p>
</subsubsect>

</subsect>

<subsect id="tpipe" tocleaf="yes">
<subhead><title><code>tpipe</code>: &tpipe-purpose;</title></subhead>

<p><code>tpipe</code> performs all kinds of general purpose manipulations
which take one table as input.
It is extremely flexible, and can do the following things
amongst others:
<ul>
<li>calculate statistics</li>
<li>display metadata</li>
<li>select rows in various ways, including algebraically</li>
<li>define new columns as algebraic functions of old ones</li>
<li>delete or rearrange columns</li>
<li>sort rows</li>
<li>convert between table formats</li>
</ul>
and combine these operations.
You can think of it as a supercharged table copying tool.
</p>

<p>The basic operation of <code>tpipe</code> is that it reads an 
input table, performs zero or more processing steps on it, 
and then does something with the output.  There are therefore
three classes of things you need to tell it when it runs:
<dl>
<dt>Input table location</dt>
<dd><p>Specified by the <code>in</code>, <code>ifmt</code> and
    <code>istream</code> parameters.
    </p></dd>

<dt>Processing steps</dt>
<dd><p>Either provide a string giving steps as the value
    of one or more <code>cmd</code> parameters, or the name of a file
    containing the steps using the <code>script</code> parameter.
    The steps that you can perform are described in 
    <ref id="filterSteps"/>.
    </p></dd>

<dt>Output table destination</dt>
<dd><p>What happens to the output table is determined by the value of
    the <code>omode</code> parameter.  
    By default, <code>omode=out</code>,
    in which case the table is written to a new table file in a format
    determined by <code>ofmt</code>.  However, you can do other things
    with the result such as
    calculate the per-column statistics (<code>omode=stats</code>),
    view only the table and column metadata (<code>omode=meta</code>),
    display it directly in TOPCAT (<code>omode=topcat</code>) etc.
    </p></dd>
</dl>
See <ref id="pipes"/> for a more detailed explanation of these ideas.
</p>

<p>The parameters mentioned above are listed in detail in the next section.
</p>

&tpipe-summary;

<subsubsect id="tpipeExamples">
<subhead><title>Examples</title></subhead>

<p>Here are some examples of <code>tpipe</code> in use with explanations
of what's going on.  For simplicity these examples assume that you have the 
<code>stilts</code> script installed and are using a Unix-like shell;
see <ref id="invoke"/> for an explanation of how to invoke the command
if you just have the Java classes.
</p>

<p>
<dl>
<dt><verbatim>
stilts tpipe cat.fits
</verbatim></dt>
<dd><p>Writes a FITS table to standard output in human-readable form.
    Since no mode specifier is given, <code>omode=out</code> is assumed,
    and output is to standard output in <code>text</code> format.
    </p></dd>

<dt><verbatim>
stilts tpipe cmd='head 5' cat.fits.gz
</verbatim></dt>
<dd><p>Does the same as the last example, but with one processing step:
    only the first five rows of the table are output.  In this case,
    the input file is compressed using gzip - this is automatically 
    detected.
    </p></dd>

<dt><verbatim>
stilts tpipe ifmt=csv xxx.csv \
             cmd='keepcols "index ra dec"' \
             omode=out ofmt=fits xxx.fits
</verbatim></dt>
<dd><p>Reads from a comma-separated values file, writes to a FITS file,
    and discards all columns in the input table apart from INDEX, RA and DEC.
    Note the quoting in the <code>cmd</code> argument: the outer quotes
    are so that the argument of the <code>cmd</code> parameter itself
    (<code>keepcols "index ra dec"</code>)
    is not split up by spaces (to protect it from the shell), 
    and the inner quotes are to keep the 
    <code>colid-list</code> argument of the 
    <code>keepcols</code> command together.
    </p></dd>

<dt><verbatim><![CDATA[
stilts tpipe ifmt=votable \
             cmd='addcol IV_SUM "(IMAG+VMAG)"' \
             cmd='addcol IV_DIFF "(IMAG-VMAG)"' \
             cmd='delcols "IMAG VMAG"' \
             omode=out ofmt=votable \
       < tab1.vot \
       > tab2.vot
]]></verbatim></dt>
<dd><p>Replaces two columns by their sum and difference in a VOTable.
    Since neither the <code>in</code> nor <code>out</code> parameters
    have been specified, the input and output are actually byte 
    streams on standard input and standard output of the 
    <code>tpipe</code> command in this case.
    The processing steps first add a column representing the sum,
    then add a column representing the difference, then delete the 
    original columns.
    </p></dd>

<dt><verbatim>
stilts tpipe cmd='addskycoords -inunit sex fk5 gal \
                               RA2000 DEC2000 GAL_LONG GAL_LAT' \
             6dfgs.fits 6dfgs+gal.fits
</verbatim></dt>
<dd><p>Adds columns giving galactic coordinates to a table.
    Both input and output tables are FITS files.
    The galactic coordinates, stored in new columns named GAL_LONG and
    GAL_LAT, are calculated from FK5 J2000.0 coordinates
    given in the existing columns named RA2000 and DEC2000.
    The input (FK5) coordinates are represented as sexagesimal strings
    (hh:mm:ss, dd:mm:ss), and the output ones are numeric degrees.
    </p></dd>

<dt><verbatim>
stilts -disk tpipe 2dfgrs_ngp.fits \
                   cmd='keepcols "SEQNUM AREA ECCENT"' \
                   cmd='sort -down AREA' \
                   cmd='head 20'
</verbatim></dt>
<dd><p>Displays selected columns for the 20 rows with largest values in
    the AREA column of a FITS table.  First the columns of interest 
    are selected, then the rows are sorted into descending order by 
    the value of the AREA column, then the first 20 rows of the resulting
    table are selected, and the result is written to standard output.
    Since a sort is being performed here, it's not possible to do all
    the processing a row at a time, since all the AREA values
    must be available for comparison during the sort.
    Two things are done here to accommodate this fact: first the
    column selection is done before the sort, so that it's only a 3-column
    table which needs to be available for random access,
    reducing the temporary storage required.
    Secondly the <code>-disk</code> flag is supplied, which means that
    temporary disk files rather than memory 
    will be used for caching table data.
</p></dd>

<dt><verbatim>
stilts tpipe 2dfgrs_ngp.fits \
             cmd='keepcols "SEQNUM AREA ECCENT"' \
             cmd='sorthead -down 20 AREA'
</verbatim></dt>
<dd><p>Has exactly the same effect as the previous example.
    However, the algorithm used by the <code>sorthead</code> filter is
    in most cases faster and cheaper on memory (only 20 rows ever have
    to be stored in this case), so this is generally a better approach
    than combining the <code>sort</code> and <code>head</code> filters.
    </p></dd>

<dt><verbatim>
stilts tpipe omode=meta cmd=&indirection.char;commands.lis http://archive.org/data/survey.vot.Z
</verbatim></dt>
<dd><p>Outputs column and table metadata about a table.
    In this case the table is a compressed VOTable at the end of a URL.
    Processing is performed according to the commands contained in a
    file named "commands.lis" in the current directory.
    </p></dd>

<dt><verbatim>
stilts tpipe in=survey.fits 
             cmd='select "skyDistanceDegrees(hmsToDegrees(RA),dmsToDegrees(DEC), \
                                             hmsToDegrees(2,28,11),dmsToDegrees(-6,49,45)) \
                          &lt; 5./60."' \
             omode=count
</verbatim></dt>
<dd><p>Counts the number of rows within a given 5 arcmin 
    cone of sky in a FITS table.
    The <code>skyDistanceDegrees</code> function is an expression which
    calculates the distance between the position specified in a row
    (as given by its RA and DEC columns) and a given point on the sky
    (here, 02:28:11,-06:49:45).
    Since <code>skyDistanceDegrees</code>'s arguments and return value are in
    decimal degrees, some conversions are required: the RA and DEC columns
    are sexagesimal strings which are converted using the
    <code>hmsToDegrees</code> and <code>dmsToDegrees</code> functions 
    respectively.  Different versions of these functions (ones which take
    numeric arguments) are used to convert the coordinates of the fixed
    point to degrees. 
    The result is compared to a constant expression representing 5 arcminutes
    in degrees.
    Any rows of the input table for which this comparison
    is true are included in the output. 
    An alternative function, <code>skyDistanceRadians</code> which works
    in radians, is also available.
    These functions and constants used here are described in detail in
    <ref id="CoordsDegrees"/> and
    <ref id="CoordsRadians"/>.
    </p></dd>

<dt><verbatim><![CDATA[
stilts tpipe ifmt=ascii survey.txt \
             cmd='select "OBJTYPE == 3 && Z > 0.15"' \
             cmd='keepcols "IMAG JMAG KMAG"' \
             omode=stats
]]></verbatim></dt>
<dd><p>Calculate statistics on the I, J and K magnitudes of selected
    objects from a catalogue.  Only those rows with the given OBJTYPE
    and in the given Z range are included.  The minimum, maximum, 
    mean, standard deviation etc of the IMAG, JMAG and KMAG columns
    will be written to standard output.
    </p></dd>

<dt><verbatim>
stilts -classpath lib/drivers/mysql-connector-java.jar \
       -Djdbc.drivers=com.mysql.jdbc.Driver \
       tpipe in=x.fits cmd="explodeall" omode=tosql \
             protocol=mysql host=localhost db=ASTRO1 dbtable=TABLEX \
             write=dropcreate user=mbt
</verbatim></dt>
<dd><p>Writes a FITS table to an SQL table, converting array-valued columns
    to scalar ones.
    To make the SQL connection work properly, the classpath is augmented
    to include the path of the MySQL JDBC driver and the
    <code>jdbc.drivers</code> system property is set to the JDBC driver
    class name.  The output will be written as a new table named TABLEX
    in the MySQL database named ASTRO1 on a MySQL server on the
    local host.  The password, if required, will be prompted for,
    as would any of the other required parameters if they had not been
    given on the command line.
    Any existing table in ASTRO1 with the name TABLEX is overwritten.
    The only processing done here is by the <code>explodeall</code> command,
    which takes any columns which have fixed-size array values and
    replaces them in the output with multiple scalar columns.
    </p></dd>

<dt><verbatim>
java -classpath stilts.jar:lib/drivers/mysql-connector-java.jar
     -Djdbc.drivers=com.mysql.jdbc.Driver \
     uk.ac.starlink.ttools.Stilts \
     tpipe in=x.fits \
           cmd=explodeall \
           omode=out \
           out="jdbc:mysql://localhost/ASTRO1?user=mbt#TABLEX"
</verbatim></dt>
<dd><p>This does exactly the same as the previous example, but achieves it
   in a slightly different way.  In the first place, java is invoked 
   directly with the necessary flags rather than getting the 
   <code>stilts</code> script to do it.  Note that you cannot use java's
   <code>-jar</code> flag in this case, because doing it like that
   would not permit access to the additional classes that contain
   the JDBC driver.
   In the second place we use <code>omode=out</code> rather than 
   <code>omode=tosql</code>.  For this we need to supply an <code>out</code>
   value which encodes the information about the SQL connection and
   table in a special URL-like format.  As you can see, this is a bit
   arcane, which is why the <code>omode=tosql</code> mode can be a help.
   </p></dd>

<dt><verbatim>
stilts tpipe USNOB.FITS cmd='every 1000000' omode=stats
</verbatim></dt>
<dd><p>Calculates statistics on a selection of the rows in a catalogue,
    and writes the result to the terminal.
    In this example, every millionth row is sampled.
</p></dd>

</dl>
</p>
</subsubsect>

</subsect>


<subsect id="tskymap" tocleaf="yes">
<subhead><title><code>tskymap</code>: &tskymap-purpose;</title></subhead>

<p><code>tskymap</code> calculates a weighted density map
(or, to put it another way, a histogram) on the sky
from columns of an input table.
The sky is divided up into some discrete set of tiles according to
a specified tessellation scheme (currently HEALPix or HTM are supported),
and the required quantities are aggregated into bins corresponding to
these tiles.
The output table has a column giving the pixel index identifying each tile,
plus one or more columns each representing an aggregation of a quantity
from the input table.
</p>

<p>By default the number of rows from the input table falling within
each tile is included as the first column in the output table.
But by specifying the <code>cols</code> and <code>combine</code> parameters
you can add more columns giving the sum, mean, median or other statistics
of input table columns or expressions as well.
</p>

<p>The output table can then, for instance, be plotted using
<ref id="plot2sky"><code>plot2sky</code></ref>'s
<ref id="layer-healpix"><code>healpix</code></ref> layer type
(though an alternative is to do that plot directly using a
<ref id="layer-skydensity"><code>skydensity</code></ref> layer).
</p>

<p>In the case of HEALPix tiling, the result can also be output
in a FITS file suitable for use by external applications that
understand the semi-standard
<webref url="&URL.HEALPIX_FITS;">FITS-Healpix</webref> convention.
Note in this case, for maximum compatibility,
the <code>fits-healpix</code> output format should in general be used.
</p>

<p>See also <ref id="tgridmap"><code>tgridmap</code></ref>,
which does the same thing for N-dimensional grid geometry.
</p>

&tskymap-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tskymap</code>:

<dl>

<dt><verbatim>
stilts tskymap in=iras_psc.fits lon=RA lat=DEC out=iras_map.csv
</verbatim></dt>
<dd><p>Writes a table representing a density map of IRAS point sources
    on the sky.  The output is a 2-column CSV file: the first column is
    pixel index, and the second column is the number of rows in the
    input table whose (RA,DEC) positions fall within that pixel.
    The default tiling is used (currently level 5 HEALPix, which has
    12288 pixels).
    </p></dd>

<dt><verbatim>
stilts tskymap in=iras_psc.fits lon=RA lat=DEC tiling=hpx6
               ocmd='healpixmeta -csys C'
               ofmt=fits-healpix out=iras_map.fits
</verbatim></dt>
<dd><p>This does a similar job to the previous example, but explicitly
    requests HEALPix level 6 tiling, and outputs to a FITS variant
    that other applications such as Aladin understand as containing
    a HEALPix map.
    The output format is specified with <code>ofmt=fits-healpix</code>,
    which writes a FITS file following the
    <webref url="&URL.HEALPIX_FITS;">HEALPix-FITS</webref> convention.
    An additional filter
    "<code><ref id="healpixmeta">healpixmeta</ref> -csys C</code>"
    is also supplied here: this indicates that the HEALPix pixels in
    the output file are to be considered in the <strong>C</strong>elestial
    (=equatorial) sky system.  If this is not done, Aladin assumes the
    coordinate system to be Galactic and will align the results incorrectly
    on the sky.
    </p></dd>

<dt><verbatim>
stilts tskymap in=2mpz.fits icmd='addskycoords fk5 galactic ra dec glon glat'
               lon=glon lat=glat tiling=hpx6
               cols='jCorr-hCorr hCorr-kCorr jCorr-kCorr' combine=median
               count=false
               complete=true
               out=2mpzColors.fits
</verbatim></dt>
<dd><p>Writes a table with columns giving median J-H, H-K and J-K colours,
    but no source count column, from the sources in the file 2mpz.fits,
    aggregated over each tile of a level 6 HEALPix grid.
    The input table has equatorial coordinates,
    but the <code>addskycoords</code> filter has been used so that the grid
    is laid on the sky with galactic coordinates.
    Setting <code>complete=true</code> guarantees that a row is written
    to the output file for every sky pixel, including empty ones.
    Note in this case that the output format is not specified explicitly,
    so it will be inferred from the filename, to be "normal" FITS,
    rather than <code>healpix-fits</code>.  The output data will still be
    present, but external applications may not automatically
    identify the pixel column.
    </p></dd>

<dt><verbatim>
stilts tskymap in=tgas_source.fits tiling=hpx7 lon=l lat=b count=false
               cols=phot_g_n_obs combine=sum-per-unit perunit=arcmin2
               ofmt=fits-healpix ocmd='healpixmeta -csys G'
               out=obs-density.fits
</verbatim></dt>
<dd><p>Calculates a level-7 HEALPix map where each cell contains the
    local number of observations per square arcminute, as determined
    by summing all the <code>phot_g_n_obs</code> values from the
    input table that fall into each tile and applying a suitable
    scaling factor.  The grid coordinate system is explicitly labelled
    as <strong>G</strong>alactic.
    </p></dd>

<dt><verbatim>
stilts tskymap in=gaia_source.colfits tiling=hpx8 lon=l lat=b count=true
               cols='bp_rp;mean phot_bp_mean_flux;sum phot_rp_mean_flux;sum'
               out=gaia-stats.fits
</verbatim></dt>
<dd><p>Simultaneously calculates the mean value of the <code>bp_rp</code>
    column and the summed values of the <code>phot_bp_mean_flux</code> and
    <code>phot_rp_mean_flux</code> columns for each HEALPix level 8 tile,
    as well as the source density.
    </p></dd>

<dt><verbatim>
stilts tskymap in=gums_lmc.fits lon=alpha lat=delta
               tiling=hpx14 complete=false omode=count
</verbatim></dt>
<dd><p>This prepares a source count map at healpix level 14
    (3 billion pixels) from the given input table.
    Since <code>complete=false</code>, only rows for non-empty pixels
    are included in the output table.  Then, since the output mode is
    <code>count</code>, these rows are just counted, discarding the
    pixels themselves, giving the number of level-14 healpix pixels
    touched by the sources in this input file.
    Note this is not necessarily the most efficient way to calculate
    coverage information.
    </p></dd>

</dl>
</p>
</subsubsect>

</subsect>


<subsect id="tskymatch2" tocleaf="yes">
<subhead><title><code>tskymatch2</code>: &tskymatch2-purpose;</title></subhead>

<p><code>tskymatch2</code> performs a crossmatch of two tables based
on the proximity of sky positions.
You specify the columns or expressions giving 
right ascension and declination in degrees for each input table, 
and a maximum permissible separation in arcseconds, and the resulting
joined table is output.
</p>

<p>If you omit expressions for the RA and Dec, an attempt is made to
identify the columns to use using column
Unified Content Descriptors 
(<webref url="http://www.ivoa.net/Documents/latest/UCD.html">UCD</webref>s)
or names.  First columns bearing appropriate UCD1 or UCD1+ values
(<code>POS_EQ_RA</code>, <code>POS_EQ_RA_MAIN</code>, 
<code>pos.eq.ra</code> or <code>pos.eq.ra;meta.main</code> and
their equivalents for declination) are sought.  If these cannot be found, 
columns named something like "RA" or "RA2000" are sought.
If either is found, the column units are consulted and radian-&gt;degree
conversions are performed if necessary (degrees are assumed if no
unit value is given).  If nothing likely can be found, then the 
command will fail with an error message.
This search logic is intended as a convenience only; it is somewhat
ad hoc and subject to change.  To make sure that the correct
angle values are being used, specify the <code>ra</code> and <code>dec</code>
position parameters explicitly.
</p>

<p><code>tskymatch2</code> is simply a cut-down version, provided for
convenience, of the more general matching task 
<ref id="tmatch2"><code>tmatch2</code></ref>. 
If you want more match options or otherwise more configurability,
you can probably find it by using <code>tmatch2</code>.
</p>

&tskymatch2-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>tskymatch2</code>:

<dl>

<dt><verbatim>
stilts tskymatch2 in1=obs_v.xml in2=obs_i.xml out=obs_iv.xml \
                  ra1=OBS_RA dec1=OBS_DEC ra2=OBS_RA dec2=OBS_DEC error=2
</verbatim></dt>
<dd><p>Takes two input catalogues (VOTables), one with observations in
    the V band and the other in the I band, and performs a match to find
    objects within 2 arcseconds of each other.
    The result is a new VOTable containing only rows where a match was found.
    </p></dd>

<dt><verbatim>
stilts tskymatch2 in1=obs_v.xml in2=obs_i.xml out=obs_iv.xml \
                  error=2
</verbatim></dt>
<dd><p>This is the same as the previous example but without explicit
    specification of the sky position columns in either table.
    It will work only if those columns are identified with appropriate
    UCDs, for instance <code>pos.eq.ra;meta.main</code> and
    <code>pos.eq.dec:meta.main</code>.
    If no suitable UCDs are in place this invocation will fail with an error.
    </p></dd>

<dt><verbatim>
stilts tskymatch2 in1=virgo1.txt ifmt1=ascii in2=mgc.fits \
                  ra1='radiansToDegrees(raRad)' dec1='radiansToDegrees(deRad)' \
                  ra2=MGC_ALPHA_J2000 dec2=MGC_DELTA_J2000 \
                  error=10 join=2not1 omode=count
</verbatim></dt>
<dd><p>Object positions in the text file virgo1.txt are compared to those
    in the FITS file mgc.fits.  The angles have been recorded in the text file
    in radians, so they are converted to degrees here before use.
    Use of the <code>join=2not1</code> parameter causes the command to
    identify all the objects in the first list 
    which do not have counterparts within 10 arcsec in the second list.
    The number of such objects found is simply output to the terminal.
    </p></dd>

</dl>
</p>
</subsubsect>

</subsect>

<subsect id="votcopy" tocleaf="yes">
<subhead><title><code>votcopy</code>: &votcopy-purpose;</title></subhead>

<p>The VOTable standard provides for three basic encodings
of the actual data within each table: TABLEDATA, BINARY and FITS.
TABLEDATA is a pure-XML encoding, which is relatively easy for humans
to read and write.
However, it is verbose and not very efficient for transmission
and processing,
for which reason the more compact BINARY format has been defined.
FITS format shares the advantages of BINARY, but is more likely to
be used where a VOTable is providing metadata 'decoration' for
an existing FITS table.
In addition, the BINARY and FITS encodings may carry their data 
either inline 
(as the base64-encoded text content of a <code>STREAM</code> element)
or externally 
(referenced by a <code>STREAM</code> element's <code>href</code> attribute).
</p>

<p>These different formats have their different advantages and
disadvantages.  Since, to some extent, programmers are humans too,
much existing VOTable software deals in TABLEDATA format even though
it may not be the most efficient way to proceed.
Conversely, you might wish to examine the contents of a BINARY-encoded 
table without use of any software more specialised than a text editor.
So there are times when it is desirable to convert from one of
these encodings to another.
</p>

<p><code>votcopy</code> is a tool which translates between these 
encodings while
making a minimum of other changes to the VOTable document.
The processing may result in some changes to lexical details 
such as whitespace in start tags, but the element structure is not
modified.  Unlike <code><ref id="tpipe">tpipe</ref></code> it does not impose 
STIL's model of what constitutes a table on the data between
reading it in and writing it out, so subtleties dependent on
the exact structure of the VOTable document will not be mangled.  
The only important changes should be the contents of
<code>DATA</code> elements in the document.
</p>

&votcopy-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Normal use of <code>votcopy</code> is pretty straightforward.
We give here a couple of examples of its input and output.
</p>

<p>Here is an example VOTable document, <code>cat.vot</code>:
<verbatim><![CDATA[
   <VOTABLE>
   <RESOURCE>

   <TABLE name="Authors">
   <FIELD name="AuthorName" datatype="char" arraysize="*"/>
   <DATA>
   <TABLEDATA>
   <TR><TD>Charles Messier</TD></TR>
   <TR><TD>Mark Taylor</TD></TR>
   </TABLEDATA>
   </DATA>
   </TABLE>

   <RESOURCE>
   <COOSYS equinox="J2000.0" epoch="J2000.0" system="eq_FK4"/>
   <TABLE name="Messier Objects">
   <FIELD name="Identifier" datatype="char" arraysize="10"/>
   <FIELD name="RA" datatype="double" units="degrees"/>
   <FIELD name="Dec" datatype="double" units="degrees"/>
   <DATA>
   <TABLEDATA>
   <TR> <TD>M51</TD> <TD>202.43</TD> <TD>47.22</TD> </TR>
   <TR> <TD>M97</TD> <TD>168.63</TD> <TD>55.03</TD> </TR>
   </TABLEDATA>
   </DATA>
   </TABLE>
   </RESOURCE>

   </RESOURCE>
   </VOTABLE>
]]></verbatim>
Note that it contains more structure than just a flat table: there are
two <code>TABLE</code> elements, 
the <code>RESOURCE</code> element of the second one being nested
in the <code>RESOURCE</code> of the first.  
Processing this document using a generic table tool such as 
<code>tpipe</code> or <code>tcopy</code> would lose this structure.
</p>

<p>To convert the data encoding to BINARY format, we simply execute
<verbatim>
   stilts votcopy format=binary cat.vot
</verbatim>
and the output is
<verbatim><![CDATA[
   <?xml version="1.0"?>
   <VOTABLE>
   <RESOURCE>

   <TABLE name="Authors">
   <FIELD name="AuthorName" datatype="char" arraysize="*"/>
   <DATA>
   <BINARY>
   <STREAM encoding='base64'>
   AAAAD0NoYXJsZXMgTWVzc2llcgAAAAtNYXJrIFRheWxvcg==
   </STREAM>
   </BINARY>
   </DATA>
   </TABLE>

   <RESOURCE>
   <COOSYS equinox="J2000.0" epoch="J2000.0" system="eq_FK4"/>
   <TABLE name="Messier Objects">
   <FIELD name="Identifier" datatype="char" arraysize="10"/>
   <FIELD name="RA" datatype="double" units="degrees"/>
   <FIELD name="Dec" datatype="double" units="degrees"/>
   <DATA>
   <BINARY>
   <STREAM encoding='base64'>
   TTUxAAAAAAAAAEBpTcKPXCj2QEecKPXCj1xNOTcAAAAAAAAAQGUUKPXCj1xAS4PX
   Cj1wpA==
   </STREAM>
   </BINARY>
   </DATA>
   </TABLE>
   </RESOURCE>

   </RESOURCE>
   </VOTABLE>
]]></verbatim>
Note that both tables in the document have been translated to BINARY format.
The basic structure of the document is unchanged: the only differences 
are within the <code>DATA</code> elements.  If we ran
<verbatim>
   stilts votcopy format=tabledata
</verbatim>
on either this output or the original input then the output would
be identical (apart perhaps from whitespace) to the input table, 
since the data are originally in TABLEDATA format.
</p>

<p>To generate a VOTable document with the data in external files,
the <code>href</code> parameter is used.  We will output in FITS format
this time.  Executing:
<verbatim>
   stilts votcopy format=fits href=true cat.vot fcat.vot
</verbatim>
writes the following to the file <code>fcat.vot</code>:
<verbatim><![CDATA[
   ...
   <DATA>
   <FITS>
   <STREAM href="fcat-1.fits"/>
   </FITS>
   </DATA>
   ...
   <DATA>
   <FITS>
   <STREAM href="fcat-2.fits"/>
   </FITS>
   </DATA>
   ...
]]></verbatim>
(the unchanged parts of the document have been skipped here for brevity).
The actual data are written in two additional files in the same
directory as the output file, <code>fcat-1.fits</code> and 
<code>fcat-2.fits</code>.  These filenames are based on the 
main output filename, but can be altered using the <code>base</code>
flag if required.  Note this has also given you FITS binary table 
versions of all the tables in the input VOTable document, which can be 
operated on by normal FITS-aware software quite separately from the VOTable
if required.
</p>

</subsubsect>

</subsect>

<subsect id="votlint" tocleaf="yes">
<subhead><title><code>votlint</code>: &votlint-purpose;</title></subhead>

<p>The VOTable standard, while not hugely complicated, has a number
of subtleties and it's not difficult to produce VOTable documents
which violate it in various ways.
In some cases the errors are small and a parser is likely to
process the document without noticing the trouble.  
In other cases, the errors are so serious that it's hard for
any software to make sense of it.
In many cases in between, different software will react in different
ways, in the worst case appearing to parse a VOTable but in 
fact understanding the wrong data.
</p>

<p><code>votlint</code> is a program which can check a VOTable document
and spot places where it does not conform to the VOTable standard,
or places which look like they may not mean what the author intended.
It is meant for use in two main scenarios:
<ol>
<li>For authors of VOTables and VOTable-producing software,
    to check that the documents they produce are legal and problem-free.
    </li>
<li>For users of VOTables (including authors of VOTable-processing software) 
    who are having problems with one and want to
    know whether it is the data or the software at fault.
    </li>
</ol>
</p>

<p>Validating a VOTable document against the VOTable schema or DTD 
of course goes a long way towards checking a VOTable document for errors,
but it by no means does the whole job, simply because the schema/DTD
specification languages don't have the facilities 
to understand the data structure
of a VOTable document.  For instance the VOTable schema 
will allow any plain text content in a <code>TD</code> element, but whether
this makes sense in a VOTable depends on the <code>datatype</code>
attribute of the corresponding <code>FIELD</code> element.  There are many
other examples.
<code>votlint</code> tackles this by parsing the VOTable document 
in a way which understands its structure and assessing the content
as critically as it can.  For any incorrect or questionable content
it finds, it will output a short message describing the problem 
and giving its location in the document.  What you do with this
information is then up to you.
</p>

<p>Using <code>votlint</code> is very straightforward.  
The <code>votable</code> argument
gives the location (filename or URL) of a VOTable document.
Otherwise, the document will be read from standard input.
Error and warning messages will be written on standard output.
Each message is prefixed with the location at which the error was
found (if possible the line and column are shown, though this is
dependent on your JVM's default XML parser).
If multiple instances of the same problem are found,
by default only a few repeats of the message are reported;
this can be controlled with the <code>maxrepeat</code> parameter.
The processing is SAX-based, so arbitrarily long tables can
be processed without heavy memory use.
</p>

<p><code>votlint</code> can't guarantee to pick up every possible
error in a VOTable document, but it ought to pick up many of the
most serious errors that are commonly made in authoring VOTables.
</p>

<p><strong>Note:</strong> <code>votlint</code>'s handling of XML namespaces
seems to be somewhat dependent on the XML parser in use.
As far as I can see, Crimson (the default in many JREs) works for any
namespace arrangements, but Xerces seems to have problems when validating
documents which use namespace prefixes.  Not sure about other parsers.
This probably won't cause you trouble, but if it does you may need to
set <code>validate=false</code> to work around it.
Contact the author if this seems to be a serious issue for you.
</p>

&votlint-summary;

<subsubsect>
<subhead><title>Items Checked</title></subhead>

<p>Votlint checks that the XML input is well-formed, and, unless the
<code>valid=false</code> parameter is supplied, that it validates against the
1.0 DTD or 1.1, 1.2, 1.3 or 1.4 schema as appropriate.
Some of the validity checks are also done by
<code>votlint</code> internally, so that some validity-type 
errors may give rise to more than one warning.  
In general, the program errs on the side of verbosity.
</p>

<p>In addition to these checks, the following checks are carried out,
and lead to ERROR reports if violations are found:
<ul>
<li><code>TD</code> contents incompatible
    <code>datatype</code>/<code>arraysize</code> attributes declared
    in <code>FIELD</code></li>
<li>BINARY/BINARY2 data streams which don't match metadata 
    declared in <code>FIELD</code></li>
<li><code>PARAM</code> values incompatible with declared 
    <code>datatype</code>/<code>arraysize</code></li>
<li>Meaningless <code>arraysize</code> declarations</li>
<li>Array-valued <code>TD</code> elements with the wrong number of elements</li>
<li>Array-valued <code>PARAM</code> values with the wrong number of 
    elements</li>
<li><code>nrows</code> attribute on <code>TABLE</code> element different
    from the number of rows actually in the table</li>
<li><code>VOTABLE</code> <code>version</code> attribute is unknown</li>
<li><code>ref</code> attributes without matching <code>ID</code> elements 
    elsewhere in the document</li>
<li>Same <code>ID</code> attribute value on multiple elements.</li>
<li>Attributes defined by external vocabularies have permitted values;
    hard-coded lists of known permitted values are stored internally,
    but the external vocabularies are consulted if necessary to check
    for values added after build time
    (currently only applies to <code>TIMESYS</code> attributes).</li>
<li>Values marked by <code>xtype</code> values as defined in DALI
    are checked for formal compliance with DALI requirements.</li>
</ul>
</p>

<p>Additionally, the following conditions, which are not actually 
forbidden by the VOTable standard, will generate WARNING reports.
Some of these may result from harmless constructions, but it is
wise at least to take a look at the input which caused them:
<ul>
<li>Wrong number of <code>TD</code> elements in row of <code>TABLEDATA</code>
    table</li>
<li>Mismatch between VOTable and FITS column metadata for
    FITS data encoding</li>
<li><code>TABLE</code> with no <code>FIELD</code> elements</li>
<li>Use of deprecated attributes</li>
<li><code>FIELD</code> or <code>PARAM</code> elements with
    <code>datatype</code> of either
    <code>char</code> or <code>unicodeChar</code>
    and undeclared <code>arraysize</code> -
    this is a common error which can result in 
    ignoring all but the first character in <code>TD</code> elements from
    a column</li>
<li><code>ref</code> attributes which reference other elements by 
    <code>ID</code> where the reference makes no, or questionable sense
    (e.g. <code>FIELDref</code> references <code>FIELD</code> in a 
    different table)</li>
<li>Multiple sibling elements (such as <code>FIELD</code>s,
    though not <code>INFO</code>s) with the
    same <code>name</code> attributes</li>
<li><code>TIMESYS</code> elements never referenced.</li>
<li><code>ucd</code> attribute values are not legal UCD1+ strings
    (if parameter <code>ucd=true</code>)</li>
<li><code>unit</code> attribute values are not legal VOUnit strings
    (if parameter <code>unit=true</code>)</li>
</ul>
</p>

</subsubsect>

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here is a brief example of running <code>votlint</code> against
a (very short) imperfect VOTable document.  If the document looks like
this:
<verbatim><![CDATA[
  <VOTABLE version="1.1">
   <RESOURCE>
    <TABLE nrows="2">
     <FIELD name="Identifier" datatype="char"/>
     <FIELD name="RA" datatype="double"/>
     <FIELD name="Dec" datatype="double"/>
     <DESCRIPTION>A very small table</DESCRIPTION>
     <DATA>
      <TABLEDATA>
       <TR>
        <TD>Fomalhaut</TD>
        <TD>344.48</TD>
        <TD>-29.618</TD>
        <TD>HD 216956</TD>
       </TR>
      </TABLEDATA> 
     </DATA> 
    </TABLE>
   </RESOURCE>
  </VOTABLE>
]]></verbatim>
then the output of a <code>votlint</code> run looks like this:
<verbatim><![CDATA[
  INFO (l.4): No arraysize for character, FIELD implies single character
  ERROR (l.7): Element "TABLE" does not allow "DESCRIPTION" here.
  WARNING (l.11): Characters after first in char scalar ignored (missing arraysize?)
  WARNING (l.15): Wrong number of TDs in row (expecting 3 found 4)
  ERROR (l.18): Row count (1) not equal to nrows attribute (2)
]]></verbatim>
(Note that the details of the reports will vary according to
the XML parser/validator that forms part of your Java installation.)
</p>

<p>Note also that the warning at line 11 has resulted from
the same error as the
one at line 4 - because the <code>FIELD</code> element has no
<code>arraysize</code> attribute, <code>arraysize="1"</code> 
(single character) is assumed,
while the author almost certainly intended <code>arraysize="*"</code>
(unknown length string).
</p>

<p>By examining these warnings you can see what needs to be done to
fix this table up.  Here is what it should look like:
<verbatim><![CDATA[
  <VOTABLE version="1.1">
   <RESOURCE>
    <TABLE nrows="1">                                <!-- change row count -->
     <DESCRIPTION>A very small table</DESCRIPTION>   <!-- move DESCRIPTION -->
     <FIELD name="Identifier" datatype="char" 
                              arraysize="*"/>        <!-- add arraysize -->
     <FIELD name="RA" datatype="double"/>
     <FIELD name="Dec" datatype="double"/>
     <DATA>
      <TABLEDATA>
       <TR>
        <TD>Fomalhaut</TD>
        <TD>344.48</TD>
        <TD>-29.618</TD>
       </TR>                                         <!-- remove extra TD -->
      </TABLEDATA>
     </DATA>
    </TABLE>
   </RESOURCE>
  </VOTABLE>
]]></verbatim>
When fed this version, <code>votlint</code> gives no warnings.
</p>

</subsubsect>

</subsect>

<subsect id="xsdvalidate" tocleaf="yes">
<subhead><title><code>xsdvalidate</code>:
                &xsdvalidate-purpose;</title></subhead>

<p><code>xsdvalidate</code> is a utility for validating XML documents
against XML Schemas.
This is not very specific to STILTS, and does not claim to do a better
job than other XSD validators out there, so if you can find another one
that suits your purposes, please use it.
It was added to the package however since it seems hard to find
suitable tools that do this job.
</p>

<p>The schema is not specified by runtime parameters,
elements in a given XML document are validated against any schemas
associated with the namespaces defined in that document.
However, it is possible to override the locations of schemas
using runtime parameters.
By using the <code>schemaloc</code> parameter, you can tell the validator
to validate against a local copy of a schema or one at a given URL,
rather than the copy that is referenced by
<code>xsi:schemaLocation</code> attributes in the document itself.
That can be convenient when developing a schema with a public namespace
that is fixed, but content which is still subject to change.
Moreover, some local copies of W3C and IVOA schemas are kept within the
application, so that external network accesses are not required during
validation.
</p>

<p>By default, successful validation results in no output and a
success exit status, but if there are validation errors then
some indication of what went wrong is printed to standard output
and the command exits with an error.
This makes it suitable for use in makefiles.
</p>

&xsdvalidate-summary;

<subsubsect>
<subhead><title>Examples</title></subhead>

<p>Here are some examples of using <code>xsdvalidate</code>:

<dl>

<dt><verbatim>
stilts xsdvalidate example-voresource.xml
</verbatim></dt>
<dd><p>Validates the given file against whatever schemas it declares.
    Success is indicated with a non-error exit.
    </p></dd>

<dt><verbatim>
stilts xsdvalidate
       doc=example-voresource.xml
       schemaloc='http://www.ivoa.net/xml/VOResource/v1.0=vor-local.xsd'
       topel='{http://www.ivoa.net/xml/RegistryInterface/v1.0}Resource'
       verbose=true
</verbatim></dt>
<dd><p>Validates the named file against whatever schemas it declares,
    but uses the local file "<code>vor-local.xsd</code>" in the current
    directory as the schema associated with the
    "<code>http://www.ivoa.net/xml/VOResource/v1.0</code>" namespace,
    rather than using any schema location given in the validated file.
    It also requires that the top-level element has the given URI and
    local name; if not, the validation will fail.
    The <code>verbose=true</code> parameter provides some diagnostics
    to standard output, including indications of which schemas were
    used and where they were acquired from.
    </p></dd>

</dl>

</p>
</subsubsect>

</subsect>

</sect>


<sect>
<subhead><title>Release Notes</title></subhead>

<p>This is STILTS, Starlink Tables Infrastructure Library Tool Set.
It is a collection of non-GUI utilites for general 
purpose table manipulation.

<dl>
<dt>Author</dt>
<dd><p>Mark Taylor (Bristol University)</p></dd>

<dt>Email</dt>
<dd><p><webref url="mailto:m.b.taylor@bristol.ac.uk"
                          >m.b.taylor@bristol.ac.uk</webref></p></dd>

<dt>WWW</dt>
<dd><p><webref url="http://www.starlink.ac.uk/stilts/"
                   >http://www.starlink.ac.uk/stilts/</webref></p></dd>
</dl>

User comments, suggestions, requests and bug reports to the above address
are welcomed.
</p>

<subsect>
<subhead><title>Acknowledgements</title></subhead>

<p>The initial development of STILTS was done under the UK's 
<webref url="http://en.wikipedia.org/wiki/Starlink_Project"
        >Starlink project</webref> (1980-2005, R.I.P.).
Since then it has been supported by
grant PP/D002486/1 from the
UK's Particle Physics and Astronomy Research Council,
the <webref url="http://www.eurovotc.org/">VOTech</webref>
project (from EU FP6),
the <webref url="http://www.astrogrid.org/">AstroGrid</webref> project
    (from PPARC/STFC),
the <webref url="http://cds.u-strasbg.fr/twikiAIDA/bin/view/EuroVOAIDA/WebHome"
            >AIDA</webref> project (from EU FP7),
grants ST/H008470/1, ST/I00176X/1, ST/J001414/1, ST/L002388/1,
ST/M000907/1, ST/R000700/1 and ST/S001980/1
from the UK's Science and Technology Facilities Council (STFC),
the <webref url="http://www.g-vo.org/">GAVO</webref> project
(BMBF Bewilligungsnummer 05A08VHA),
the European Space Agency,
and the EU FP7-2013-Space project GENIUS.
All of this support is gratefully acknowledged.
</p>

<p>Apart from the excellent Java 2 Standard Edition itself,
the following external libraries provide important parts of STILTS's
functionality:
<ul>
<li><webref url="http://www.gnu.org/software/jel/">JEL</webref>
    (GNU) for algebraic expression evaluation</li>
<li><webref url="https://github.com/cds-astro/cds-healpix-java"
            >cds-healpix-java</webref>
    (CDS) for HEALPix manipulation</li>
<li><webref url="http://www.1t3xt.com/">iText</webref>
    (1T3XT BVBA) for PDF output</li>
<li><webref url="http://www.jfree.org/jfreesvg/">JFreeSVG</webref>
    for SVG output</li>
<li><webref url="http://www.jibble.org/epsgraphics/">EPSGraphics2D</webref>
    (Jibble) for encapsulated postscript output</li>
<li><webref url="http://cds.u-strasbg.fr/resources/doku.php?id=moc">MOC</webref>
    (CDS) for Multi-Order Coverage map manipulation</li>
<li><webref url="http://cdsportal.u-strasbg.fr/adqltuto/">ADQL</webref>
    (CDS) for ADQL parsing in TAP query preparation</li>
<li><webref url="https://skyview.gsfc.nasa.gov/jar/jar.html"
            >Skyview in a Jar</webref> (NASA) for sky axis drawing</li>
<li><webref url="http://forge.scilab.org/index.php/p/jlatexmath/"
            >JLaTeXMath</webref> (Scilab) for LaTeX typesetting in plots</li>
<li><webref
     url="http://www.acme.com/java/software/Acme.JPM.Encoders.GifEncoder.html"
            >GifEncoder</webref>
    (Acme) for GIF output</li>
<li><webref url="http://www.sdss.jhu.edu/htm/">HTM</webref>
    (Sloan Digital Sky Survey) for HTM-based celestial sphere row matching
    (now deprecated within STILTS)</li>
</ul>
Thanks in particular to Nickolai Kouropatkine and Chris Stoughton 
of Fermilab for writing the PixTools specially for use in STIL.
</p>

<p>Many people have contributed ideas and advice to the development of
STILTS and its related products.
A list of some of them can be found in the TOPCAT user document, SUN/253.
</p>

<p>If you use this software in published work, the following citation
would be appreciated:
<blockquote>
<webref url="http://adsabs.harvard.edu/abs/2006ASPC..351..666T"
                                          >2006ASPC..351..666T</webref>:
M. B. Taylor, "STILTS - A Package for Command-Line Processing of Tabular Data",
in Astronomical Data Analysis Software and Systems XV,
eds. C. Gabriel et al., ASP Conf. Ser. 351, p. 666 (2006)
</blockquote>
</p>

</subsect>


<subsect id="versions">
<subhead><title>Version History</title></subhead>

<p>Releases to date have been as follows:
<dl>

<dt id="v0.1b">Version 0.1b (29 April 2005)</dt>
<dd><p>First public release
    </p></dd>

<dt id="v0.2b">Version 0.2b (30 June 2005)</dt>
<dd><p>
  <ul>
  <li>Added Times func class for MJD-ISO8601 time conversions.</li>
  <li>Fixed bug when doing NULL_ test expressions on first column in table.</li>
  </ul>
</p></dd>

<dt id="v1.0b">Version 1.0b (30 September 2005)</dt>
<dd><p>This is the first non-experimental release of STILTS, and 
    it incorporates major changes and backward incompatibilities 
    since version 0.2b.
    </p>
    <p><dl>
       <dt>Parameter system</dt>
       <dd><p>The parameter system has undergone a complete rewrite;
           there is now only a single command "<code>stilts</code>",
           invoked using the <code>stilts</code> script or the 
           <code>stilts.jar</code> jar file, and the various tasks are
           named as subsequent arguments on the command line.
           Command arguments are supplied after that.
           The new invocation syntax is described in detail elsewhere in 
           this document.  As well as invocation features such
           as improved on-line help, optional prompting, 
           parameter defaulting, and more uniform access to common features,
           this will make it more straightforward to wrap these tasks 
           for use in non-command-line environments, such as behind a
           SOAP or CORBA interface, or in a CEA-like execution environment.
           </p></dd>

        <dt>Crossmatching</dt>
        <dd><p>A new command <code>tmatch2</code> has been introduced.
            This provides flexible and efficient crossmatching between
            two input tables.  Future releases will provide commands for
            intra-table and multi-table matching.
            </p></dd>

        <dt>Concatentation</dt>
        <dd><p>A new command <code>tcat</code> has been introduced, which
            allows two tables to be glued together top-to-bottom.
            This is currently working but very rudimentary - improvements
            will be forthcoming in future releases.
            </p></dd>

        <dt>Calculator</dt>
        <dd><p>A new utility command <code>calc</code> has been introduced,
            which performs one-line expression evaluations from the 
            command line.
            </p></dd>

        <dt>Pipeline filters</dt>
        <dd><p>The following new filter commands for use in <code>tpipe</code>
            and other commands have been introduced:
            <ul>
            <li><code>addskycoords</code>: calculates new 
                celestial coordinate pair from existing ones 
                (FK4, FK5, ecliptic, galactic, supergalactic)</li>
            <li><code>replacecol</code>: replaces column data,
                using existing metadata</li>
            <li><code>badval</code>: replaces given 'magic'
                value with null</li>
            <li><code>replaceval</code>: replaces given 'magic'
                value with any specified value</li>
            <li><code>tablename</code>: edits table name</li>
            <li><code>explodecols</code> and <code>explodecols</code> commands
                replace <code>explode</code></li>
            </ul>
        </p>
        <p>The new <code>stream</code> parameter of <code>tpipe</code> now
        allows you to write filter commands in an external file, to 
        facilitate more manageable command lines.
        </p>
        <p>Wildarding for column specification is now allowed for some
        filter commands.
        </p></dd>

        <dt>Algebraic functions</dt>
        <dd><p><ul>
            <li>New functions for converting time values between different
                coordinate systems (Modified Julian Date, ISO-8601,
                Julian Epoch and Besselian Epoch).</li>
            <li>New RANDOM special function.</li>
        </ul></p></dd>

        <dt>Documentation</dt>
        <dd><p>SUN/256 has undergone many changes.  Much of the tool
            documentation is now automatically generated from the code
            itself, which goes a long way to ensuring that the documentation
            is correct with respect to the current state of the code.
            </p></dd>

    </dl></p>
</dd>

<dt id="v1.0-1b">Version 1.0-1b (7 October 2005)</dt>
<dd><p>Fixed jar file manifest bug which prevented working on Java 1.5</p></dd>

<dt id="v1.1">Version 1.1 (10 May 2006)</dt>
<dd><p>A number of new features and capabilities have been introduced:
    <dl>
    <dt><code>tcube</code> Command</dt>
    <dd><p>The new <ref id="tcube" plaintextref="yes">tcube</ref> command
        calculates N-dimensional histograms (density maps) from N columns
        of an input table and writes the result to a FITS file.</p></dd>

    <dt>Processing Filters</dt>
    <dd><p>The following new <ref id="filterSteps">filters</ref>
        have been added:
        <ul>
        <li><code>stats</code> filter provides the same information as
            the old <code>stats</code> output mode, but allows much more
            flexible use of the results.  It can also calculate many new
            quantities, including quantiles, skew and kurtosis.</li>
        <li><code>meta</code> filter provides the same information as
            the old <code>meta</code> output mode, but allows much more
            flexible use of the results.</li>
        <li><code>assert</code> filter provides in-pipeline logical
            assertions.</li>
        <li><code>uniq</code> filter collapses multiple adjacent identical 
            or similar rows.</li>
        <li><code>sorthead</code> filter provides a (usually) more
            efficient method of doing what you could previously do 
            by combining <code>sort</code> and <code>head</code> filters.</li>
        <li><code>colmeta</code> filter adds/modifies metadata for selected
            columns.</li>
        <li><code>check</code> filter checks table in stream - for debugging
            purposes only.</li>
        </ul>
        </p>
        <p>Additionally usage of the <code>sort</code> filter has been changed
        so that it can now do everything that <code>sortexpr</code> used to
        be able to do; <code>sortexpr</code> is now withdrawn.
        </p></dd>

    <dt>Output Modes</dt>
    <dd><p>The following new <ref id="outModes">output modes</ref>
        have been introduced:
        <ul>
        <li><code>plastic</code> mode broadcasts the table to
            one or all registered PLASTIC listeners.</li>
        <li><code>cgi</code> mode writes the table to standard output in a
            form suitable for output from a CGI script.</li>
        <li><code>discard</code> mode throws away the table.</li>
        </ul>
        and usage of the following has been modified:
        <ul>
        <li><code>topcat</code> mode now attempts to use PLASTIC 
            (amongst other methods) to contact TOPCAT.</li>
        <li><code>stats</code> and <code>meta</code> modes are mildly
            deprecated in favour of the corresponding new filters
            (see above).</li>
        </ul>
        </p></dd>

    <dt>Other new features</dt>
    <dd><p>
        <ul>
        <li>New IPAC table format input handler added.</li>
        <li>New <code>csv-noheader</code> format variant output handler
            added.</li>
        <li><code>roundDecimal</code> and <code>formatDecimal</code>
            functions introduced for more control over visual appearance
            of numeric values.</li>
        <li>Experimental facilities for automatically generating a CEA
            application description file.</li>
        </ul>
        </p></dd>

    <dt>Bug fixes and minor improvements</dt>
    <dd><p>
        <ul>
        <li>Now copes with 'K'-format FITS binary table columns
           (64-bit integers).</li>
        <li>Improved, though still imperfect, retention of table-wide
            metadata in VOTables.</li>
        <li>Distinctions between null and false values in boolean columns are
            handled more carefully for FITS and VOTable files.</li>
        <li>Efficiency improvement when writing FITS-plus format
            (now only requires a maximum of two passes rather than
            three of the input rows).</li>
        <li>Added the <code>mark.workaround</code> 
            <ref id="sysProperties">system property</ref> which can 
            optionally work around a bug in some input streams 
            ("Resetting to invalid mark" errors).</li>
        <li>Fixed a bug in Cartesian matching which failed to match
            if the required error in any dimension was zero.</li>
        <li>Fixed erroneous reports about unknown <code>ucd</code> and 
            <code>utype</code> attributes of TABLE element in 
            <code>votlint</code>.</li>
        <li>When joining tables, column name comparison to determine
            whether deduplication is required is now case-insensitive.</li>
        <li>Error message improved when no automatic format detection
            is attempted for streamed tables.</li>
        <li>Setting <code>istream=true</code> is now less likely to cause a
            "Can't re-read stream" error.</li>
        </ul>
        </p></dd>
    </dl>

</p></dd>

<dt id="v1.2">Version 1.2 (7 July 2006)</dt>
<dd><p>
    <dl>
    <dt>Column-oriented Storage</dt>
    <dd><p>New features for permitting column-oriented storage
        (<code>colfits</code> format, new <code>startable.storage</code> 
        policy "<code>sideways</code>") have been introduced.
        These can provide considerable efficiency improvements for
        certain tasks when working with very large (and especially wide)
        tables.
        </p></dd>
    <dt>New VO commands</dt>
    <dd><p>Added two new commands for querying Virtual Observatory services:
        <ul>
        <li><code>multicone</code>
            - Makes multiple cone search queries to the same service</li>
        <li><ref id="regquery"><code>regquery</code></ref>
            - &regquery-purpose;</li>
        </ul>
        These tasks are experimental and may be modified or renamed in
        future releases.
        </p></dd>
    <dt>Other items</dt>
    <dd><p>
        <ul>
        <li><code>transpose</code> filter added.</li>
        <li>Added flux conversion functions (Jansky&lt;-&gt;magnitude).</li>
        <li>ISO-8601 strings now permit times of 24:00:00 as they should.</li>
        </ul>
        </p></dd>
    </dl>
</p></dd>

<dt id="v1.2-1">Version 1.2-1 (3 August 2006)</dt>
<dd><p>
    <ul>
    <li>Tab-Separated Table (TST) format now supported for reading and 
        writing.</li>
    <li>New <code>setparam</code> and <code>clearparams</code> filters.</li>
    <li>Added ICRS coordinate system for <code>addskycoords</code>.</li>
    <li>TUCDnn header cards now used in FITS files to transmit UCDs
        (non-standard mechanism).</li>
    <li>Efficiency improvements for column-oriented access.</li>
    </ul>
    </p></dd>

<dt id="v1.3">Version 1.3 (5 October 2006)</dt>
<dd><p>
    <dl>

    <dt>Table Concatenation</dt>
    <dd><p>The old <code>tcat</code> command has been replaced by more capable 
        <ref id="tcat"><code>tcat</code></ref> and
        <ref id="tcatn"><code>tcatn</code></ref> commands.
        Between them these provide concatenation of an unlimited number of
        homogeneous or heterogeneous input tables.
        Additional columns may be added to indicate which of the input tables
        given output rows originated from.
        </p></dd>

    <dt>Parameter value indirection</dt>
    <dd><p>Certain parameters 
        (<code>in</code> in <ref id="tcat"><code>tcat</code></ref>,
        <code>cmd</code> and friends)
        may now be specified in the form "&indirection.char;filename".
        This indicates that the value for the parameter is to be obtained
        by reading it from the named file.
        This is useful if a very long value is required for the parameter
        in question.  The <code>script</code> parameter of
        <ref id="tpipe"><code>tpipe</code></ref> has therefore been
        withdrawn, since it did just the same thing.
        </p></dd>

    <dt>MySpace access</dt>
    <dd><p>Direct access to the MySpace virtual file system is now provided
        by use of <code>ivo:</code>- or <code>myspace:</code>-type URLs.
        </p></dd>

    <dt>Conversion functions</dt>
    <dd><p>
        <ul>
        <li>Time conversion functions between MJD and Decimal Year have
            been added
            (<ref id="Times"/>).</li>
        <li><code>toHex</code> and <code>fromHex</code> numeric conversion
            functions have been added
            (<ref id="Conversions"/>).</li>
        </ul>
        </p></dd>

    <dt>Documentation improvements</dt>
    <dd><p>
        <ul>
        <li>The HTML version of SUN/256 now uses CSS to provide better 
            highlighting of <code>commands</code> etc.</li>
        <li>The <ref id="outModes">Output Modes</ref> and
            <ref id="filterSteps">Processing Filter</ref> sections are
            now split into subsections to make the table of contents
            clearer.</li>
        <li>The <ref id="cmdUsage">Command Reference</ref> section now 
            has only one level of subsection listed in the table of
            contents to make it clearer.</li>
        </ul>
        </p></dd>

    <dt>Other new features and improvements</dt>
    <dd><p>
        <ul>
        <li>Added <code>-J</code> flag to <code>stilts</code> script
            for passing flags directly to Java.</li>
        <li>Added new <code>out</code> parameter to <code>votlint</code>.</li>
        <li>Added <code>-ifndim</code> and <code>-ifshape</code> flags to
            <ref id="explodeall">explodeall</ref> filter.</li>
        <li>The <code>exact</code> match mode in 
            <ref id="tmatch2"><code>tmatch2</code></ref>
            now copes with array-valued columns.</li>
        <li>Added <code>force</code> parameter to <code>multicone</code> task 
            as a workaround for some broken services.</li>
        <li>Added Sample (as opposed to Population) Standard Deviation/Variance
            calculation options to the <code>stats</code> filter.</li>
        <li>Improved CEA description file output - now contains details of 
            all tasks rather than just a few, as well as various 
            improvements in documentation etc.</li>
        </ul>
        </p></dd>

    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Fixed erroneous complaints from <code>votlint</code> about 
            <code>utype</code> attribute on RESOURCE elements.</li>
        <li>Fixed a couple of minor crossmatching bugs (which wouldn't
            have affected results).</li>
        </ul>
        </p></dd>

    </dl>
</p></dd>

<dt id="v1.3-1">Version 1.3-1 (Starlink Hokulei release)</dt>
<dd><p>
    <ul>
    <li>New command <ref id="tjoin"><code>tjoin</code></ref> introduced.</li>
    <li>Output to MySpace can now be streamed, if running under J2SE1.5 or
        later.</li>
    <li>Slight changes to parameters for <code>votlint</code>
        and <code>votcopy</code>.</li>
    <li>Fixed bug in handling of single quotes in FITS file metadata.</li>
    <li>Added <code>-bench</code> flag to <code>stilts</code> command.</li>
    <li>Various scalability improvements for use with very large (Tb?) 
        files.</li>
    <li>Improved efficiency for <code>text</code> and <code>ascii</code>
        output formats (now one-pass not two-pass).</li>
    <li>Improved CEA app-description file, including especially option lists
        for things like input and output formats.</li>
    <li>Added README.cea file to distribution.</li>
    <li>Fixed problem which could mis-report VOTable out of memory errors
        as Broken Pipe.</li>
    <li>Added Vega&lt;-&gt;AB magnitude conversion constants to 
        <ref id="Fluxes">Fluxes</ref> functions.</li>
    <li>Added <code>duptag</code> parameters to <ref id="tmatch2">tmatch2</ref>
        task for customised renaming of columns with duplicated names.</li>
    <li>Added hyperbolic trig functions to
        <ref id="Maths">Maths</ref> class
        (<code>sinh</code>, <code>cosh</code>, <code>tanh</code>
        and inverses).</li>
    <li>Added cosmology distance calculations in class
        <ref id="Distances">Distances</ref>.</li>
    <li>Added <ref id="funcs"><code>funcs</code></ref> task, a browser for
        expression language function documentation.</li>
    <li>Added <code>-checkversion</code> to list of <code>stilts</code>
        <ref id="stilts-flags">flags</ref>.</li>
    </ul>
    </p></dd>

<dt id="v1.3-2">Version 1.3-2 (6 July 2007)</dt>
<dd><p>
    <ul>
    <li>Added optional <code>table</code> parameter to 
        <ref id="calc"><code>calc</code></ref> command
        (for access to table parameters).</li>
    <li>Can use table parameter names in expressions using 
        <code>&param.prefix;</code> notation (<ref id="jel-paramref"/>).</li>
    <li>Can reference columns/parameters by UCD by using 
        <code>&ucd.prefix;</code> notation 
        in expressions (<ref id="jel-colref"/>)
        and as column identifiers (<ref id="col-id"/>).</li>
    <li>Improved deduplication of column names when joining tables.</li>
    <li>Fix error in output of FITS table <code>TNULL</code><m>n</m> header
        cards - write them as numeric not string values.</li>
    <li>Improve error message for broken CSV files.</li>
    <li>Modified JDBC handling so that MySQL and PostgreSQL do not run out
        of heap memory when streaming large datasets for input.
        Think I've done the same for SQL Server, but this is not tested.</li>
    <li>Improve error reporting in the presence of a deficient JVM
        (such as GNU <code>gcj</code>).</li>
    <li>Add locale-specific <code>formatDecimalLocal</code> functions in 
        class <ref id="Formats">Formats</ref>.</li>
    <li>Add <code>fluxToLuminosity</code> and <code>luminosityToFlux</code>
        functions in class 
        <ref id="Fluxes">Fluxes</ref>.</li>
    <li>Fix bug which was causing NullPointerExceptions 
        in the <code>transpose</code> filter.</li>
    </ul>
    </p></dd>

<dt id="v1.3-3">Version 1.3-3 (4 Sep 2007)</dt>
<dd><p>
    <ul>
    <li>Experimental, and currently undocumented, <code>sqlcone</code> 
        task introduced, along with some classes in package 
        <code>uk.ac.starlink.ttools.cone</code> designed for library use by
        AstroGrid DSA code.
        </li>
    <li>CEA description of <code>tmatch2</code> <code>params</code>
        parameter now has <code>minoccurs=0</code>, since that can be true
        for exact matches.</li>
    </ul>
    </p></dd>

<dt id="v1.3-4">Version 1.3-4 (10 Sep 2007)</dt>
<dd><p>
    <ul>
    <li>Fixed VotCopy bug.</li>
    </ul>
    </p></dd>

<dt id="v1.3-5">Version 1.3-5 (30 Oct 2007)</dt>
<dd><p>
    <ul>
    <li>Added <code>-stdout</code> and <code>-stderr</code> flags to
        <code>stilts</code> command.</li>
    <li>Some bugs fixed in generation of CEA <code>app-description.xml</code>
        file.</li>
    <li>Documentation provided for <code>sqlcone</code> command.</li>
    <li>Fixed error in <code>fluxToLuminosity</code> function.</li>
    </ul>
    </p></dd>

<dt id="v1.4">Version 1.4 (6 December 2007)</dt>
<dd><p>
    <dl>
    <dt>Table joins</dt>
    <dd><p>This version provides more cross matching functionality.
        Added to the existing <ref id="tmatch2"><code>tmatch2</code></ref>
        command are new tasks:
        <ul>
        <li><ref id="tskymatch2"><code>tskymatch2</code></ref>:
            stripped down version of <code>tmatch2</code> for ease of use when
            matching with sky coordinates.</li>
        <li><ref id="tmatch1"><code>tmatch1</code></ref>:
            internal matcher, finds groups of objects within a table.</li>
        <li><ref id="tmatchn"><code>tmatchn</code></ref>:
            finds group or multiple-pair matches between multiple (&gt;2)
            tables.</li>
        </ul>
        </p>
        <p>Two tasks have been renamed for improved clarity and consistency:
        <ul>
        <li><code>multicone</code> is now named
            <ref id="coneskymatch"><code>coneskymatch</code></ref></li>
        <li><code>sqlcone</code> is now named
            <ref id="sqlskymatch"><code>sqlskymatch</code></ref></li>
        </ul>
        </p>
        <p>There has also been some enhancement and rationalisation of
        parameters for all table join tools (<code>tmatch*</code> as well as 
        <code>tjoin</code>,
        <code>coneskymatch</code> and <code>sqlskymatch</code>):
        <ul>
        <li>All table join commands now use similar <code>fixcols</code> and 
            <code>suffix*</code> parameters to control renaming of duplicated
            columns in output tables (note this replaces the old 
            <code>duptag*</code> parameters in <code>tmatch2</code>).</li>
        <li>Crossmatching tasks have a new <code>progress</code> parameter
            which allows you to configure whether progress is reported to the 
            console.</li>
        <li>The <code>copycols</code> parameter of <code>coneskymatch</code> 
            and <code>sqlskymatch</code> now defaults to "<code>*</code>" 
            (include all columns from input table in the output).</li>
        </ul>
        </p>
        <p><ref id="match"/> of the manual has been somewhat
        rearranged and improved.
        </p></dd>

    <dt>Other enhancements</dt>
    <dd><p>
        <ul>
        <li>FITS reader now imports table HDU header cards as table parameters.
            </li>
        <li>CeaWriter can now output CEA service definition XML config file as
            well as app-description file (experimental - may be withdrawn).</li>
        </ul>
        </p></dd>

    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Embedded spaces in output ASCII format table column names
            are now substituted with underscores.</li>
        <li>Fix a bug which caused an infinite number of dots to be printed
            when attempting a crossmatch with an empty input table.</li>
        <li>Corrected <code>votlint</code> handling of TABLEDATA-type 
            multi-dimensional <code>char</code>/<code>unicodeChar</code> arrays.
            These are now split up into strings by counting characters rather
            than using whitespace delimiters.
            I <em>think</em> it's doing the right thing now.</li>
        </ul>
        </p></dd>

    </dl>
    </p></dd>

<dt id="v1.4-1">Version 1.4-1 (28 January 2008)</dt>
<dd><p>
    <dl>
    <dt>New RDBMS-related features</dt>
    <dd><p>
        <ul>
        <li>New command <ref id="sqlclient"><code>sqlclient</code></ref>,
            which is a general JDBC-based SQL command-line client.</li>
        <li>New command <ref id="sqlupdate"><code>sqlupdate</code></ref>,
            which allows updates to existing rows in SQL tables.</li>
        <li>Some changes to <ref id="mode-tosql"><code>tosql</code></ref> 
            output mode:
            <ul>
            <li>choice of options for how to write to the database output table,
                controlled by new associated parameter <code>write</code>
                (can be <code>create</code>, <code>dropcreate</code> 
                     or <code>append</code>)</li>
            <li>associated parameter <code>newtable</code> renamed
                <code>dbtable</code></li>
            <li>associated parameter <code>database</code> renamed
                <code>db</code> for consistency with other commands</li>
            </ul>
            </li>
        </ul>
        </p></dd>
    <dt>Local and service-based matching command enhancements</dt>
    <dd><p>
        <ul>
        <li>New parameter <code>scorecol</code> added to
            <code>tmatch2</code>,
            <code>coneskymatch</code> and
            <code>sqlskymatch</code> commands,
            which controls adding a new column to match output tables 
            containing a goodness-of-match value.</li>
        <li>New parameter <code>parallel</code> added to 
            <ref id="coneskymatch"><code>coneskymatch</code></ref> task
            which allows multiple cone searches to be carried out in 
            parallel.</li>
        <li>New parameter <code>erract</code> added to 
            <code>coneskymatch</code> which controls response to 
            isolated failures in individual cone search queries.</li>
        </ul>
        </p></dd>
    <dt>General improvements</dt>
    <dd><p>
        <ul>
        <li>Improved error reporting (reasons for errors are now reported
            even without the <code>-debug</code> flag).</li>
        <li>Add new help option <code>help='*'</code> which prints help for
            all parameters of a task at once.</li>
        <li>Added (mostly undocumented) <code>+verbose</code> flag 
            for reducing verbosity level.</li>
        <li>Minor improvements to CEA app-description.</li>
        <li>Downgraded from WARNING to INFO log messages about 
            the (extremely common) VOTable syntax error of omitting 
            a FIELD/PARAM element's <code>datatype</code> attribute.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v1.4-2">Version 1.4-2 (26 March 2008)</dt>
<dd><p>
    <dl>
    <dt>Minor enhancements:</dt>
    <dd><p>
        <ul>
        <li>Add <code>progress</code> parameter to <code>tmatchn</code>.</li>
        <li>Add <code>emptyok</code> parameter to 
            <code>coneskymatch</code>.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fixed pair matching performance bug (slower if tables were not 
            given in the right order) introduced at v1.4.</li>
        <li>Fixed null handling error in <code>calc</code> task.</li>
        <li>Fixed error in <code>stats</code> filter cardinality value 
            calculation.</li>
        <li>Fixed minor bugs in suffix addition for matching commands
            <code>fixcols</code>.</li>
        <li>Removed unformatted XML output in <code>stats</code> filter
            usage message.</li>
        <li>Try to avoid exponential format in cone search URLs 
            (some endpoints seem to require fixed point format).</li>
        <li>Minor CEA fixes.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.0b">Version 2.0b (23 October 2008)</dt>
<dd><p>This version contains two new major items, plotting and server mode.
    Both work, but are missing desirable features and have not had extensive
    testing in the field, so should be considered experimental at this stage.
    <dl>
    <dt>Plotting</dt>
    <dd><p>Two table plotting commands are now provided:
        <ul>
        <li><ref id="plot2d"><code>plot2d</code></ref>:
            &plot2d-purpose;</li>
        <li><ref id="plot3d"><code>plot3d</code></ref>:
            &plot3d-purpose;</li>
        <li><ref id="plothist"><code>plothist</code></ref>:
            &plothist-purpose;</li>
        </ul>
        See also the new <ref id="plot" plaintextref="yes">Plotting</ref>
        section in the manual.
        </p></dd>
    <dt>Server/Servlet Mode</dt>
    <dd><p>A new command <ref id="server"><code>server</code></ref> is
        provided which allows STILTS commands to be executed via HTTP.
        One purpose of this is to facilitate server-side use of the 
        plotting commands co-located with data to generate on-the-fly 
        graphical summaries of server-held datasets.
        </p></dd>
    <dt>Smaller enhancements and bugfixes</dt>
    <dd><p>
        <ul>
        <li>Efficiency improvements (~25%? in both CPU time and memory usage) 
            for HEALPix-based sky crossmatching 
            (thanks to Nikolay Kouropatkine at Fermilab for a new version of 
            the PixTools library).</li>
        <li>New class <ref id="Arrays">Arrays</ref> 
            added to algebraic functions.</li>
        <li>New Appendix <ref id="classified" plaintextref="yes"
                              >Commands by Category</ref>
            added to manual.
            </li>
        <li>Add <code>minReal</code> and <code>maxReal</code> functions 
            (max/min ignoring blank values) in class
            <ref id="Arithmetic">Arithmetic</ref>.
            </li>
        <li>Sexagesimal field identification for ASCII input files 
            is less stringent
            (now permits minutes or seconds equal to 60).</li>
        <li>Minor CEA fixes.</li>
        <li>HEALPix bug fix
            (<webref url="http://home.fnal.gov/~kuropat/HEALPIX/PixTools.html"
                     >PixTools</webref> bug fix update).</li>
        <li>Fix bug in use of <code>tcat</code>'s
            <code>loccol</code> parameter.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.0-1">Version 2.0-1 (23 December 2008)</dt>
<dd><p>
    <ul>
    <li>Can reference columns/parameters by Utype by using 
        <code>&utype.prefix;</code> notation
        in expressions (<ref id="jel-colref"/>)
        and as column identifiers (<ref id="col-id"/>).</li>
    <li>Non-alphanumeric column names may now be used for algebraic expressions
        in the special case that the expression is just the value of a
        single column.</li>
    <li><code>regquery</code> command has changed in implementation,
        data access, and output format.  It now queries VOResource1.0 
        registries rather than the very out of date registry protocol which
        was used in earlier versions.</li>
    </ul>
    </p></dd>

<dt id="v2.0-2">Version 2.0-2 (9 January 2009)</dt>
<dd><p>
    <ul>
    <li>Added new <ref id="mode-samp">samp</ref> output mode which passes
        the generated table to clients using the SAMP protocol.</li>
    <li>Updated the <ref id="mode-topcat">topcat</ref> output mode to use
        SAMP as one way of communicating with a running TOPCAT.</li>
    <li><code>-version</code> flag now reports starjava subversion revision
        as well as other items.</li>
    </ul>
    </p></dd>

<dt id="v2.0-3">Version 2.0-3 (27 March 2009)</dt>
<dd><p>
    <ul>
    <li>Fits BINTABLE TZERO/TSCAL value reading improvements:
       <ul>
       <li>Columns with integer TZERO values now read as integers
           rather than floating point values where possible.  
           This includes unsigned longs ('K'), which were previously 
           represented as doubles with lost precision.  
           Unsigned longs which are too large however (&gt;2<sup>63</sup>) 
           are read as nulls.</li>
       <li>Byte-valued columns can now be written out by 
           <code>fits-basic</code> output
           handler as signed byte values (TFORM=B,TZERO=-128) 
           rather than signed shorts (TFORM=I).</li>
       <li>More comprehensive testing.</li> 
       <li>Fixed bug in calculating value scaled double ('D') values.</li>
       <li>Fixed bug in typing value for scaled float ('E') arrays.</li>
       <li>Fixed bug which caused registry queries (<code>regquery</code>) 
           to fail for Java 1.6.</li>
       </ul></li>
    <li>Fix minor bugs in detail of <code>votlint</code>'s validation tests
        (VOTABLE element content model, 
        INFO and PARAM and FIELD required attributes).</li>
    <li>Report application name and version in User-Agent header of outgoing
        HTTP requests.</li>
    <li>The fixed length Substring Array Convention for string arrays
        (<code>TFORMnn=rAw</code>) 
        is now understood for FITS binary tables.</li>
    <li>Minor SAMP bugs fixed (JSAMP upgraded to 0.3-1).</li>
    </ul>
    </p></dd>

<dt id="v2.0-4">Version 2.0-4 (17 July 2009)</dt>
<dd><p>
    <ul>
    <li>Work around J2SE mark/reset bug when loading table direct from URL.</li>
    <li>Produce null rather than nonsense results from sky coordinate
        conversions with unphysical latitudes
        (<code>addskycoords</code> filter).</li>
    <li>Produce null rather than questionable results from sexagesimal
        conversions with mins/secs out of range.</li>
    <li>Fix two bugs in <code>votcopy</code>: XML processing instructions 
        garbled on output, and pathnames in <code>base</code> parameters
        inappropriately flattened in hrefs attribute values.</li>
    </ul>
    </p></dd>

<dt id="v2.0-5">Version 2.0-5 (2 Oct 2009)</dt>
<dd><p>
    <ul>
    <li>VOTable 1.2 supported.</li>
    <li><code>votlint</code> can now validate VOTable documents following
        the (provisional, 2009-09-29 PR) VOTable 1.2 standard.</li>
    <li>Namespacing of VOTable documents made more intelligent, and
        configurable using the <code>votable.namespacing</code> system
        property.</li>
    <li><code>votlint</code> now checks that the correct XML namespaces are
        in use.</li>
    <li>Be more careful in XML, including VOTable, output; 
        fix VOTable output encoding to be UTF-8,
        and ensure no illegal XML characters are written.</li>
    <li>HTML table output is now HTML 4.01 by default 
        (includes THEAD and TBODY tags).</li>
    <li><code>parse*</code> string-&gt;numeric conversion 
        functions now cope with leading or trailing whitespace.</li>
    <li>Work around illegally truncated type declarations in IPAC tables.</li>
    <li>Fix a bug which caused the first table in a multi-table file
        (FITS or VOTable) to be used in streaming mode, even if
        a subsequent one was requested.</li>
    <li>Bug fixed in crossmatching output: entries which should have been 
        null were sometimes written as non-null (typically large negative
        numbers) in FITS and in non-TABLEDATA VOTable output.
        This affected cells in otherwise non-nullable columns
        where the entire row was absent.  The previous behaviour is not
        likely to have been mistaken for genuine results.
        </li>
    </ul>
    </p></dd>

<dt id="v2.1">Version 2.1 (6 November 2009)</dt>
<dd><p>
    <ul>
    <li><code>coneskymatch</code> can now match using SIA and SSA
        services as alternatives to Cone Search ones
        (see its new <code>servicetype</code> parameter).</li>
    <li>Fixed an obscure bug which could under rare circumstances cause
        truncation of strings with leading/trailing whitespace read
        from text-format files.</li>
    <li>A new <code>startable.storage</code> policy "<code>adaptive</code>"
        is now the default.  This should mean running out of memory less
        often.  The old behaviour can be restored by giving the new
        <code>-memory</code> command line flag.</li>
    </ul>
    Note that the STIL API used by this release has changed in some
    backwardly incompatible ways, and may change further.
    If you're using STILTS as a library rather than an application
    you might want to wait for a later release when the API has settled down.
    </p></dd>

<dt id="v2.1-1">Version 2.1-1 (21 December 2009)</dt>
<dd><p>
    <ul>
    <li>Plotting commands can now output to PDF as well as existing graphics
        formats.</li>
    <li>New filter <ref id="fixcolnames"><code>fixcolnames</code></ref>.</li>
    <li>Fixed internationalisation bug which could cause
        <code>coneskymatch</code> to fail in locales that use ","
        for a decimal point.</li>
    <li>Significant performance improvements related to the case of VOTable
        documents containing many tables.</li>
    </ul>
    </p></dd>

<dt id="v2.1-2">Version 2.1-2 (24 March 2010)</dt>
<dd><p>
    <ul>
    <li><ref id="jystilts">JyStilts</ref> introduced.
        This is a jython (i.e. Python, though not CPython)
        interface to the STILTS commands.
        It is believed to be fully working, but somewhat experimental -
        feedback is encouraged.</li>
    <li>Considerable performance and scalability improvements to the
        crossmatching commands
        (<code>tmatch1</code>, <code>tmatch2</code>, <code>tmatchn</code> and
         <code>tskymatch2</code>).
        For several common regimes, using default settings, 
        memory use has been decreased by a
        factor of about 5, and CPU time reduced by a factor of about 3.</li>
    <li>Add optional tuning parameters to crossmatch commands
        (parameter <code>tuning</code> for 
        <code>tmatch1</code>, <code>tmatch2</code> and <code>tmatchn</code>,
        and parameter <code>healpixk</code> for <code>tskymatch2</code>).
        Experimentation with these can lead to significant performance
        improvements for given matches.</li>
    <li>Fixed a crossmatch bug; it was giving a possibility of
        suboptimal "<code>find=best</code>"
        match assignments when pair matching in crowded fields.
        Crossmatch results thus may differ between earlier versions
        and this one.  Both are reasonable, but the newer behaviour is 
        more correct.  In non-crowded fields, there should be no change.</li>
    <li>Further performance improvement for VOTable documents with
        very many TABLEs.</li>
    <li>Memory management adjusted further - default (Adaptive) storage policy 
        now uses direct allocation (=<code>malloc()</code>) 
        for intermediate-sized buffers
        to avoid running out of java heap space.</li>
    <li>New option "<code>find=each</code>" for
        <code>coneskymatch</code> and <code>sqlskymatch</code> commands.
        This allows you to get an output table with exactly one row for
        each row of the input table.</li>
    <li>New flag <code>-memgui</code> to monitor memory usage during runs.</li>
    <li>Add new filter <ref id="rowrange"><code>rowrange</code></ref>.</li>
    <li>Add new functions to 
        <ref id="Arrays">Arrays</ref>:
        <code>array</code> functions for constructing arrays,
        and new aggregating functions <code>median</code> and
        <code>quantile</code>.</li>
    <li>Syntax of the crossmatching commands' <code>progress</code> parameter
        has changed; it now has an additional option which will write
        limited profiling information as well as logging as the match
        progresses.</li>
    <li>Add <code>ylabel</code> parameter to <code>plothist</code> command.</li>
    <li>The <code>random</code> and <code>sequential</code> filters have
        been renamed <code>randomview</code> and <code>seqview</code>
        respectively.  This provides a better idea of what they do.
        Since they are only useful for debugging, it is unlikely that this
        will break anyone's existing code.</li>
    <li>New filter <code>random</code> introduced which converts tables
        to random-access if necessary.</li>
    <li>Document previously undocumented <code>legend</code> parameter to
        plotting commands.</li>
    <li>Matching commands <code>matcher</code> parameters can now accept
        classnames of <code>MatchEngine</code> implementation classes as
        an option.</li>
    <li>Classes are now distributed as a zip of jars 
        (<code>stilts_jars.zip</code>) as an alternative to
        the monolithic jar file (<code>stilts.jar</code>).
        This may be more appropriate for those using STILTS classes in a
        framework that contains other third party class libraries.</li>
    <li>Adjusted the way that data types are read from JDBC databases.
        Date, Time and Timestamp typed columns will now be converted to Strings
        which means they can be written to most output formats
        (previously they were omitted from output tables).</li>
    <li>STILTS no longer attempts to communicate with TOPCAT using SOAP.
        TOPCAT's SOAP interface has been deprecated since v2.1 (2006),
        so this isn't likely to cause trouble, and it permits removal of
        SOAP (Axis) classes from the application jar file, saving several
        megabytes and reducing potential version clash problems.</li>
    <li>Fix bug in code for handling very large mapped FITS files.
        This was causing fatal read errors in some cases.</li>
    </ul>
    </p>
    </dd>

<dt id="v2.2">Version 2.2 (6 August 2010)</dt>
<dd><p>New capabilities for multi-table I/O have been introduced:
    <ul>
    <li>New multi-table output tasks
        <code>tmulti</code> and <code>tmultin</code>.
        These currently just copy multiple input tables to a single 
        multi-table container file (e.g. Multi-Extension FITS or 
        multi-TABLE VOTable).  Future releases may generalise the output of
        multi-table processing.</li>
    <li>New <code>multi</code> parameter introduced for <code>tcat</code>
        and <code>tmulti</code> tasks to pick up all tables in a 
        multi-table container file.</li>
    <li>New JyStilts functions <code>treads</code> and <code>twrites</code>
        for multi-table I/O.</li>
    </ul>
    There are some additional enhancements:
    <ul>
    <li>Added experimental name-resolution filter <code>addresolve</code>;
        this currently uses Sesame.</li>
    <li>Added filter <code>repeat</code>, which repeats table rows a
        given number of times.</li>
    </ul>
    And a number of bug fixes:
    <ul>
    <li>Recognise unofficial column type "<code>long</code>" in IPAC format
        tables.</li>
    <li>Better behaviour (warn + failover) when attempting to read large
        files on 32-bit OS or JVM.</li>
    <li>Efficiency warning now issued for large compressed FITS files.</li>

    <li>Upgraded PixTools HEALPix library to 2010/02/09 version.
        This fixes a bug that could theoretically cause deficient crossmatch
        results, though I haven't managed to produce such errors.</li>
    <li>Fixed bug in TST table output.</li>
    <li>Fixed bug in FITS-plus metadata output
        (table parameters were getting lost).</li>
    <li>Corrected literature references in Fluxes conversion class
        documentation (thanks to Mattia Vaccari).</li>
    <li>Fixed bug in CSV file parsing that could ignore header row in
        absence of non-numeric columns.</li>
    <li>Shape and ElSize metadata items now correctly reported by 
        <code>meta</code> filter.</li>
    <li>Fixed JyStilts bug when supplying an empty string for a parameter
        value.</li>
    </ul>
    </p>
    <p>Finally, from this release STILTS requires version 1.5 (a.k.a. 5.0)
    of the Java J2SE Runtime Environment; it will no longer run on 
    version 1.4, which is now very old.  I don't expect this to cause
    compatibility issues for anyone, but I'm interested to hear if that's
    not the case.
    </p></dd>

<dt id="v2.2-1">Version 2.2-1 (23 December 2010)</dt>
<dd><p>
    <ul>
    <li>Storage management improvements; removed restriction on large
        (&gt;2Gb) non-FITS datasets in some circumstances.</li>
    <li>Efficiency improvement in sequential mapped access to large FITS
        files.</li>
    <li>Fix so FITS tables &gt;2Gb can provide random access in 32-bit mode
        (though slower than 64-bit).</li>
    <li>FITS files now store table names in EXTNAME (and possibly EXTVAR)
        header cards.</li>
    <li>Window placement for the few GUI tasks should now behave a bit more
        like platform norms, rather than sitting in the top left hand
        corner.</li>
    <li>HTML table output now writes cell contents which look like URLs
        in HTML &lt;A&gt; tags.</li>
    <li>Basic authorization (<code>http://user:pass@host/path</code>) on
        table URLs handled.</li>
    <li>Fixed file pointer int overflow bug in FITS MultiMappedFiles.</li>
    </ul>
    </p></dd>

<dt id="v2.3">Version 2.3 (9 May 2011)</dt>
<dd><p>
    <dl>
    <dt>TAP</dt>
    <dd><p>The new commands
        <ref id="tapquery"><code>tapquery</code></ref> and
        <ref id="tapresume"><code>tapresume</code></ref> have been introduced.
        These provide support for the Table Access Protocol (TAP),
        and allow freeform queries in an SQL-like language to be
        made to remote databases.
        </p></dd>
    <dt>Minor enhancements</dt>
    <dd><p>
        <ul>
        <li>Random Groups HDUs are now tolerated, though not interpreted, 
            within FITS files.</li>
        <li>Added <code>soapout</code> parameter to <code>regquery</code>
            command.</li>
        <li>Added <code>count</code>, <code>variance</code> and
            <code>stdev</code> functions to 
            <ref id="Arrays">Arrays</ref>.</li>
        <li>Upgrade to JSAMP v1.2.</li>
        <li>Improve text rendering in <code>funcs</code> window display.</li>
        <li>Attempt case-sensitive matching before case-insensitive for
            column names.</li>
        <li>Fix <code>replaceval</code> filter to work with Infinities.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes and workarounds</dt>
    <dd><p>
        <ul>
        <li>JDBC table input handler now effectively downcasts
            BigInteger/BigDecimal types to Long/Double.  
            The PostgreSQL JDBC driver seems to use the Big* types routinely 
            for numeric values (which I don't think it used to do).</li>
        <li>Add workaround for J2SE bug
            <webref
             url='http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4795134'
                   >#4795134</webref>,
             which could cause errors when reading compressed FITS files.</li>
        <li>Fix FITS character handling bug which could cause corrupted FITS
            files on output in presence of non-ASCII characters.</li>
        <li>Fix (some) JDBC connection leaks.</li>
        <li>Add missing parameters <code>dashNS</code> and
            <code>linewidthNS</code> to <code>plot2d</code> task.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.3-1">Version 2.3-1 (30 June 2011)</dt>
<dd><p>
    <ul>
    <li>Added new command <code>taplint</code>.
        This is a validator for TAP (Table Access Protocol) services.
        It is only likely to be useful to people developing or operating
        TAP services.</li>
    <li>ASCII table parsers now understand python-friendly
        <code>nan</code> and <code>inf</code> representations.</li>
    <li>Added new constants to expression language
        <code>Infinity</code> and <code>NaN</code>.</li>
    <li>Fixed a significant bug in sky crossmatching.
        If all points in a table were on one side of the RA=0 line,
        but the error radius extended across that line, matches on the
        other side could be missed.  Matches could also be missed if
        different tables used different conventional ranges for RA
        (e.g. -180..180 in one case and 0..360 in another).
        This fix may in some, but not most, cases result in slower matching
        than previously.</li>
    <li>Fixed <code>coneskymatch</code> cone search verbosity parameter so that
        VERB=3 is not erroneously ignored.</li>
    </ul>
    </p></dd>

<dt id="v2.4">Version 2.4 (27 October 2011)</dt>
<dd><p>
    <dl>
    <dt>Crossmatching:</dt>
    <dd><p>
        <ul>
        <li>Two new asymmetric match options <code>best1</code>
            and <code>best2</code> have been added
            for the <code>find</code> parameter in the pair matching commands 
            <code>tmatch2</code> and <code>tskymatch2</code>.
            They correspond to finding the best match in table B for each row
            in table A, and in crowded fields often provide more intuitive
            semantics than the previous symmetric <code>best</code> option
            (in non-crowded fields there is generally no difference).
            This replicates the matching performed by some other tools,
            including Aladin.</li>
        <li>New matchers have been added to permit matching of general
            elliptical, rather than just circular, regions in both planar
            and sky coordinates; see 
            <ref id="EllipseCartesianMatchEngine"><code>2d_ellipse</code></ref>,
            and
            <ref id="EllipseSkyMatchEngine"><code>skyellipse</code></ref>.</li>
        <li>Another new matcher is available for dealing with per-object
            errors in Cartesian coordinates (previously per-object errors
            could only be handled in sky coords); see
            <ref id="ErrorCartesianMatchEngine"
                 ><em>N</em><code>d_err</code></ref>.
            </li>
        <li>Semantics of the
            <ref id="SkyMatchEngine-err"><code>skyerr</code></ref>
            matcher have changed slightly.</li>
        </ul>
        </p></dd>
    <dt>Expression language functions:</dt>
    <dd><p>
        <ul>
        <li>Algebraic functions involving angles are now mostly available
            using degrees as well as radians.
            The <code>Coords</code> class has been replaced by
            <ref id="CoordsDegrees"><code>CoordsDegrees</code></ref> and
            <ref id="CoordsRadians"><code>CoordsRadians</code></ref>
            classes providing sky coordinate functions,
            and a new class
            <ref id="TrigDegrees"><code>TrigDegrees</code></ref>
            provides normal degree-based trigonometric functions
            alongside the radian-based versions in
            <ref id="Maths"><code>Maths</code></ref>.
            Some of the old function names have changed to make clear that they
            use radians and not degrees.
            This change should be much more convenient in most cases;
            sorry it's taken so long to get round to.</li>
        <li>Add new <code>join</code> function is added to the
            <ref id="Arrays">Arrays</ref> class
            to combine all the elements of an array into a string.</li>
        </ul>
        </p></dd>
    <dt>taplint:</dt>
    <dd><p>There are several bugfixes and changes related to the
           TAP validator tool <code>taplint</code>, mostly thanks to bug
           reports etc from the TAP community:
        <ul>
        <li>Improve test logic for record limiting queries.</li>
        <li>Errors no longer reported (e.g. E-Qxx-CNAM) 
            for unexpected TAP_SCHEMA table column ordering
            (when running query stage but no metadata acquisition stages).</li>
        <li>Add new stage MDQ, which checks query result columns for all
            tables against declared metadata.</li>
        <li>Add check of versioned and unversioned LANG variants.</li>
        <li>Now uses corrected upload ID 
            (<code>ivo://ivoa.net/std/TAPRegExt#upload-*</code>)
            as per most recent TAPRegExt draft.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes and minor enhancements:</dt>
    <dd><p>
        <ul>
        <li>Add parameter <code>parse</code> to <code>tapquery</code> command,
            allowing pre-send syntax checking of submitted ADQL.</li>
        <li>Add experimental system properties
            <code>star.basicauth.user</code> and 
            <code>star.basicauth.password</code>.</li>
        <li>Improve resilience of <code>coneskymatch</code> in the presence of
            unreliable or inconsistent DAL services.</li>
        <li>A PARAMref element with no referent in a VOTable
            no longer causes an uncaught NullPointerException.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.5">Version 2.5 (28 March 2013)</dt>
<dd><p>
    <dl>
    <dt>New coverage-related functionality:</dt>
    <dd><p>
        <ul>
        <li>Add new command <ref id="pixsample"><code>pixsample</code></ref>
            which can sample pixel data from HEALPix table files
            (useful for things like Schlegel dust extinction).
            Also <ref id="addpixsample"><code>addpixsample</code></ref> filter,
            which does the same job.</li>
        <li>Add new command <code><ref id="pixfoot">pixfoot</ref></code>
            which can generate MOC (Multi-Order Coverage) maps.</li>
        <li>Add MOC-based coverage filter to <code>coneskymatch</code>
            when using some Cone Search services (mostly VizieR).
            This uses the Multi-Order Coverage map service operated by CDS.
            It can make VizieR multi-cone queries much faster by not doing
            cone searches that are outside the coverage region of the
            catalogue in question.</li>
        <li>Add new class 
            <ref id="Coverage">Coverage</ref>
            to the expression language containing MOC-related functions
            (currently, just <code>inMoc</code>).</li>
        </ul>
        </p></dd>
    <dt>Other new capabilities:</dt>
    <dd><p>
        <ul>
        <li>Add IPAC table output format.</li>
        <li>Add new class
            <ref id="KCorrections">KCorrections</ref>
            to the expression language,
            containing a method for calculating K-corrections
            following the method of Chilingarian and Zolotukhin.</li>
        <li>You can now reference tables in multi-extension FITS files by name
            (EXTNAME or EXTNAME-EXTVER) as an alternative to by HDU index.</li>
        </ul>
        </p></dd>
    <dt>VOTable enhancements:</dt>
    <dd><p>
        <ul>
        <li>VOTable input, output and validation are now supported
            for version 1.3 of the VOTable standard.</li>
        <li>The version of the VOTable format used for VOTable output
            can now be selected,
            by using the <ref id="sysProperties">system property</ref>
            <code>votable.version</code>.
            Output version is VOTable 1.2 by default.</li>
        <li><code>votlint</code> has been changed so that it handles
            different VOTable versions more capably.
            Versions 1.1+ are now validated against a schema
            (which is how those versions are defined) rather than against
            a DTD hacked to do the same job as the schema.
            VOTable 1.3 validation is now provided.</li>
        <li>The <code>votcopy</code> command has a new <code>version</code>
            parameter to control output version,
            and a new <code>nomagic</code> parameter to control whether
            <code>VALUES</code>/<code>null</code> attributes are removed
            where appropriate.</li>
        <li>Infinite floating point values are now correctly encoded
            in VOTable output ("<code>+Inf</code>"/"<code>-Inf</code>",
            not "<code>Infinity</code>"/"<code>-Infinity</code>" as in
            previous versions).</li>
        <li><code>votlint</code> is now stricter about floating point
            <code>TD</code> element contents.</li>
        <li>VOTable output no longer writes the <code>schemaLocation</code>
            attribute by default.</li>
        </ul>
        </p></dd>
    <dt>Other enhancements:</dt>
    <dd><p>
        <ul>
        <li>Add new function <code>hypot</code> (=sqrt(x*x+y*y))
            to the <code>Maths</code> class in expression language.</li>
        <li>Add new <code>split</code> functions for string splitting
            to the <code>Strings</code> class in expression language.</li>
        <li>Add <code>-utype</code> flags for <code>addcol</code>,
            <code>replacecol</code>, <code>colmeta</code> and
            <code>setparam</code> filters,
            and <code>utype</code> option for <code>meta</code> filter.</li>
        <li>Some changes to the <code>toString</code> function:
            it now works on non-numeric values,
            gives the right answer for <code>Long</code> integers
            and character values,
            and returns a blank value rather than the string "null" or "NaN"
            for blank inputs.</li>
        <li>Sexagesimal to numeric angle conversion functions now permit the
            seconds part of the sexagesimal string to be missing.</li>
        <li>Changes to the IPAC format definition are accommodated:
            the "long"/"l" type, which is apparently now official,
            no longer generates a warning, and headers may now use
            minus signs instead of whitespace.</li>
        <li>Add <code>OBS</code> stage (ObsTAP validation)
            to <code>taplint</code>.</li>
        <li>Add more checks to <code>CAP</code> stage of <code>taplint</code>.
            Declared languages (including features) and output formats are
            now checked.</li>
        <li>Tidy up error reporting a bit (fewer duplicate nested messages
            reported).</li>
        <li>PNG graphics output no longer has transparent background.</li>
        <li>Issue a warning for high values of <code>coneskymatch</code>
            <code>parallel</code> parameter.</li>
        <li>Upgrade JSAMP library to version 1.3-3.</li>
        <li>Upgrade Gr&#x00e9;gory Mantelet's ADQL library to version 1.1.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes:</dt>
    <dd><p>
        <ul>
        <li>Fix serious and long-standing bug
            (bad TZERO header, causes subsequent reads to fail)
            for FITS output of boolean array columns.</li>
        <li>Fix small but genuine sky matching bug.
            The effect was that near the poles matches near the specified
            threshold could be missed.  The bug was in the PixTools library,
            fixed at the 2012-07-28 release.</li>
        <li>Fix bug in <code>tmatchn</code> group mode which could result in
            output rows with columns from only a single table,
            i.e. not representing an inter-table match, 
            even when <code>join*=default</code>.</li>
        <li>Fix bug which failed when attempting to read FITS files with
            complex array columns (<code>TFORMn=rC/rM</code>).</li>
        <li>Fix failure when caching very large sequential tables.</li>
        <li>Fix bug in <code>replacecol</code> and <code>replaceval</code>
            filters which could cause truncation of strings in FITS
            and possibly VOTable output when the new value was longer
            than the previously declared maximum length.</li>
        <li>Fix <code>tcat</code>, <code>tcatn</code> so that in most cases
            output column metadata is compatible with all input tables, not
            just the first one in terms of nullability, array shape etc.</li>
        <li>Adjust SQL writer to avoid a type error for MySQL.</li>
        <li>Fix bug in HMS sexagesimal formatting: minus sign was omitted from
            negative angles.  Now the output is forced positive.</li>
        <li>Cope with 1-column CSV files.</li>
        <li>Use the correct form "<code>rows</code>"/"<code>bytes</code>"
            rather than "<code>row</code>"/"<code>byte</code>"
            for TAP capability unit values.</li>
        <li>Fix error bar rendering bug which could result in diagonal lines
            being offset near the edge of plots.</li>  
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.5-1">Version 2.5-1 (1 July 2013)</dt>
<dd><p>
    <dl>
    <dt>New functionality</dt>
    <dd><p>
        <ul>
        <li>Add read-only support for
            <webref url="https://cdf.gsfc.nasa.gov/">CDF</webref>
            (NASA Common Data Format) files.</li>
        <li>Add Median Absolute Deviation calculation
            (<code>MedAbsDev</code> and <code>ScMedAbsDev</code>)
            options to <ref id="stats"><code>stats</code></ref> filter.</li>
        <li>Improved handling of HTTP basic authorization.
            401s now generate a useful message about the
            <code>star.basicauth.*</code> system properties if they
            have not been set up.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes and minor enhancements</dt>
    <dd><p>
        <ul>
        <li>Fix CSV regression bug introduced at v2.5
            - CSV files now work again with MSDOS-style line breaks.</li>
        <li>Fixed FITS output bug which could result in badly-formed
            string-valued header cards (no closing quote).</li>
        <li>Source code is now managed by git and not subversion.
            The format of the "Starjava revision" string reported by the
            <code>-version</code> flag has changed accordingly.</li>
        <li>Output mode <code>meta</code> now copes better with array-valued
            table parameters.</li>
        <li>Implemented fixes to reduce the chance of users inadvertently
            overloading external Cone/SIA/SSA services with multicone-like
            queries.
            First, fix it so that abandoned queries are properly terminated,
            rather than continuing to hit the server until completion or JVM
            shutdown.
            Second, implement a sensible default maximum value for the
            <code>parallel</code> parameter of <code>skyconematch</code>
            (though this may be adjusted with a system property).</li>
        <li>Quoting behaviour has changed when generating SQL to write to
            RDBMS tables.  This ought to reduce problems related to mixed-case
            identifiers.  However, it is possible that it could lead to 
            unforseen new anomalies.</li>
        <li>More <code>toString</code> overloads - now works for byte and
            boolean values too.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v2.5-2">Version 2.5-2 (7 March 2014)</dt>
<dd><p>
    <ul>
    <li>Add some more colour maps.</li>
    <li>Fix some broken and misdocumented non-table-output JyStilts
        commands (<code>tcube</code>, <code>pixfoot</code>).</li>
    <li>Fix bug which prevented access to long integer array elements
        from expression language.</li>
    <li>The Exact matcher now considers scalar numeric values equal if
        they have the same numeric value; they are no longer required
        to have the same type.</li>
    <li>Fixed a registry client bug which means that the
        <code>regquery</code> command can now successfully talk to
        the NVO/VAO/STSci registry.
        That has been broken since mid-2010.</li>
    <li>Add new command <code>tloop</code> for generating single-column
        tables from a numeric loop variable.</li>
    <li><code>taplint</code> now checks for the right ObsCore ID,
        though still recognises the wrong one (got from TAPRegExt),
        and warns if found.</li>
    <li>Fix TST input handler so TST files with fewer than 3 columns
        can be read.</li>
    <li>Add <code>Nd_cuboid</code> matcher option to match commands.</li>
    </ul>
    </p></dd>

<dt id="v2.5-3">Version 2.5-3 (4 July 2014)</dt>
<dd><p>
    <dl>
    <dt>New and improved functionality:</dt>
    <dd><p>
        <ul>
        <li>Add new command
            <ref id="cdsskymatch"><code>cdsskymatch</code></ref>.
            In most cases (for querying tables that can be found in VizieR)
            this can and should be used instead of
            <code>coneskymatch</code> - it's <em>much</em> faster.</li>
        <li>Commands <code>coneskymatch</code>, <code>sqlskymatch</code>
            and <code>pixfoot</code> will now guess RA/Dec columns
            if relevant parameters are left blank.</li>
        <li>Added new graphics output format <code>png-transp</code>
            to generate PNG files with transparent backgrounds.</li>
        <li>Upgraded Gregory Mantelet's ADQL library to version 1.2.
            Better ADQL parsing.</li>
        </ul>
        </p></dd>
    <dt>Improvements and adjustments to
        <ref id="taplint"><code>taplint</code></ref>:</dt>
    <dd><p>
        <ul>
        <li>Rework <code>taplint</code>
            API to facilitate static acquisition of
            report codes during programmatic use.
            A few error codes have changed.</li>
        <li>Add new "duff query" test to <code>taplint</code>.</li>
        <li>Avoid <code>taplint</code> MDQ stage data type mismatch error
            report for BOOLEAN/boolean declared/returned data.</li>
        <li><code>taplint</code> now takes steps to ensure that TAP_SCHEMA
            column list query is not truncated.</li>
        <li><code>taplint</code> now flags absence of ObsCore table with
            I[nfo] not F[ailure] status.</li>
        <li>Change the implementation of <code>taplint</code> stages which
            perform validation against XSD schemas.
            Schemas from external namespaces may now be imported and used.
            The CPV stage, which was previously broken and disabled
            by default, is now fixed and enabled by default.
            Known/expected schemas are stored locally, and a warning is
            reported if external ones are used.  Schema validation seems
            remarkably complicated, so it's possible there are still errors
            in this implementation - if you suspect so,
            please report it.</li>
        <li>Add missing geometric reserved words to ADQL reserved word
            list.  This fixes some problems with column names like
            "DISTANCE" in taplint tests.</li>
        <li>Fixed some bugs related to TAP table uploads.
            In particular these could cause incorrect table upload error
            reports in <code>taplint</code>.</li>
        </ul>
        </p></dd>

    </dl>
    </p></dd>

<dt id="v3.0">Version 3.0 (3 October 2014)</dt>
<dd><p>
    <dl>
    <dt>New plotting commands:</dt>
    <dd><p>A set of new plotting commands are provided which give
        comprehensive access to all the new-style visualisation capabilities
        available in TOPCAT v4.  These commands are
        documented in <ref id="plot2"/>.
        These commands, and the underlying visualisation facilities,
        are considerably more capable than the, now deprecated,
        old-style plot commands
        <code>plot2d</code>, <code>plot3d</code> and <code>plothist</code>.
        </p></dd>
    <dt>Programmatic invocation:</dt>
    <dd><p>Programmatic invocation of STILTS tasks from third-party java code
        is now officially sanctioned and documented in the new
        <ref id="taskApi"/>.
        To support this changes have been made to the parameter system 
        (<code>Parameter</code> class now supports generics)
        and there are some visible
        changes to the user documentation as well
        (parameters now report their data type,
        and tasks report their classname).
        Normal (e.g. command-line) usage should not undergo any changes,
        but a fair bit of UI code has changed, so unexpected problems
        are possible.
        </p></dd>
    <dt>Other items:</dt>
    <dd><p>
        <ul>
        <li>Add new output mode
            <code><ref id="mode-gui">gui</ref></code>,
            which displays the table data in a scrollable
            window on the screen.</li>
        <li>Add new <code>-allowunused</code> flag to the stilts command.
            If this is set, then unused parameter settings on the command line
            just result in a warning, not failure of the command.</li>
        <li>Attempting to write FITS tables with &gt;999 columns now fails
            with a more helpful error message.</li>
        <li>Improved Unicode handling in VOTables.
            Fixed a serious bug in <code>votcopy</code>
            that generated unreadable output to BINARY or BINARY2 serialization
            if any non-empty column had datatype="unicodeChar".
            Also improved behaviour when copying between tables with
            unicodeChar columns; these are usually preserved now, rather
            than being squashed to datatype char.
            Some lurking Unicode-related issues remain.</li>
        <li>The TAP client now tolerates whitespace around
            UWS status codes.</li>
        <li>taplint: downgrade unknown post-table QUERY_STATUS value message
            from Error to Warning.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-1">Version 3.0-1 (13 November 2014)</dt>
<dd><p>
    <dl>
    <dt>New functionality:</dt>
    <dd><p>
        <ul>
        <li>Add (experimental) read-only support for Gaia/DPAC GBIN format.</li>
        <li>Add new task
            <ref id="tapskymatch"><code>tapskymatch</code></ref>.</li>
        <li>Functions in class
            <ref id="Coverage">Coverage</ref>
            adjusted:
            new function <code>nearMoc</code>, and MOC can be identified by
            VizieR table IDs as well as by filename/URL.</li>
        <li>For <ref id="repeat"><code>repeat</code></ref> filter,
            add <code>-row|-table</code>
            flags to control sequence of output rows.</li>
        <li>For <ref id="setparam"><code>setparam</code></ref>
            and <ref id="repeat"><code>repeat</code></ref> filters,
            allow use of an algebraic expression for values,
            not just a literal value.</li>
        <li>Add special values <code>$ncol</code> and <code>$nrow</code> to
            the 
            <ref id="jel-paramref">expression language</ref>
            to refer to the column and row counts in a table.
            The special variable <code>index</code> is also deprecated in favour
            of <code>$index</code> or <code>$0</code>.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes and minor improvements:</dt>
    <dd><p>
        <ul>
        <li>Add some more colour maps for aux/density shading.</li>
        <li>Fix <code>stilts</code> invocation script to pick up classes from
            <code>stilts.jar</code> in script directory in preference to
            other places (e.g. topcat-full.jar).</li>
        <li>Fix <code>taplint</code> to permit application/xml
            not just text/xml content-type where appropriate (UWS stage).</li>
        <li>Fix <code>taplint</code> so it doesn't warn (W-TMV-UNSC) about
            unknown VOSITables schema.</li>
        <li>Fix <code>taplint</code> so that <code>unicodeChar</code>
            matches CHAR/VARCHAR in the same way as <code>char</code>
            for column type declaration purposes.</li>
        <li>Fix <code>taplint</code> so that capabilities document can have
            TAPRegExt dataModel ivo-id elements with xs:anyURI rather than
            vr:IdentifierURI (only a warning is issued in the latter case),
            in anticipation of TAPRegExt-1.0 Erratum #1.</li>
        <li>Adjust <code>taplint</code> to handle adql:TIMESTAMP columns more
            carefully on upload and retrieval.</li>
        <li>Update JSAMP to v1.3.5.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-2">Version 3.0-2 (6 February 2015)</dt>
<dd><p>
    <dl>
    <dt>Plotting enhancements:</dt>
    <dd><p>
        <ul>
        <li>Linear fitting of points is now available using the 
            <ref id="layer-linearfit">linearfit</ref> layer type for
            <code>plot2plane</code>.  Points may be weighted.</li>
        <li>You can add titles to plots using the new <code>title</code>
            parameter.</li>
        <li>New plot layer type
            <ref id="layer-sizexy"><code>sizexy</code></ref>
            allows plotting (optionally autoscaled)
            markers with horizontal and vertical extents
            independently determined by input data.</li>
        <li>More flexibility when assigning colour maps, in 
            <ref id="shading-aux">aux</ref> and
            <ref id="shading-density">density</ref> shading modes, and
            <ref id="layer-spectrogram">spectrogram</ref> layer.
            New parameters <code>*func</code> allow assignment of
            different data-&gt;ramp mapping functions
            (sqrt and square as well as linear and logarithmic),
            and new parameters <code>*quant</code> allow quantisation
            of the colour map to discrete levels.</li>
        <li>Replace <code>maxsizeN</code> parameter with <code>autoscaleN</code>
            for <ref id="layer-size"><code>size</code></ref> plot layer type.
            You can now optionally turn off autoscaling and specify marker size
            in pixels instead.</li>
        <li>Add <code>auxcrowd</code> parameter to plot2 tasks to influence
            tick crowding on aux axis colour ramp.  Also adjust default to use
            fewer ticks.</li>
        <li>Add some "dart" options (fixed-base open or filled triangles)
            for plotting vectors (see <code>arrowN</code> parameter in
            layers like
            <ref id="layer-xyvector"><code>xyvector</code></ref>).</li>
        <li>Add some "triangle" options (variable-base open or filled triangles)
            for plotting ellipses (see <code>ellipseN</code> parameter in layers
            like <ref id="layer-xyellipse"><code>xyellipse</code></ref>).</li>
        <li>Histogram normalisation option adjusted
            so that total area under bars,
            rather than total height of bars, is fixed.</li>
        <li>The <code>PlotDisplay</code> class that forms the result of
            plot2 commands can now have <code>PointSelectionListener</code>s
            registered on it.  This lets you determine what point a user
            has clicked on if you're using the plotting classes from
            third party java code.</li>
        </ul>
        </p></dd>
    <dt>FITS I/O:</dt>
    <dd><p>
        <ul>
        <li>Reworked part of the FITS table input implementation, 
            in particular adjusting the way memory mapping is done to reduce
            resource requirements on some platforms.
            If you notice any difference, it should be reduced virtual and
            perhaps resident memory usage, and some (~10%?) performance 
            improvements, when reading large FITS/colfits files.
            If you were previously having problems
            with large memory allocations leading to
            disk thrashing and system lockup when scanning files
            larger than RAM (this didn't happen on all OSes),
            these will hopefully have gone away.
            However, please report anything that appears to be working worse
            than before, or continued memory usage issues.</li>
        <li>Colfits files can now be accessed from streams,
            not just uncompressed disk files
            (though that's not necessarily a good idea).</li>
        </ul>
        </p></dd>
    <dt>Bugfixes and workarounds:</dt>
    <dd><p>
        <ul>
        <li>Fixed a query bug (missing <code>REQUEST=queryData</code> parameter)
            in the multi-SSA mode (<code>servicetype=ssa</code>) of
            <code>coneskymatch</code>.
            This long-standing bug would have stopped this command working
            at all with well-behaved SSA services.</li>
        <li>Fixed error in fits-var output
            (PCOUNT header card did not include block alignment gap).</li>
        <li>Graphics coordinates are now calculated in floating point 
            rather than as integers.  This fixes problems that could 
            cause scaled vectors, ellipses etc to be drawn with shapes
            or orientations badly wrong due to rounding errors.
            It also improves plotting of analytic functions, especially
            to vector contexts (PDF/EPS).</li>
        <li>Fix some problems to do with zooming to very large/small
            plot axis ranges.</li>
        <li>Hide error bars (etc) that would extend to negative values
            on logarithmic axes; previously they were being drawn in anomalous
            places.</li>
        <li>Fix NullPointerException bug when null value was supplied to
            multi-word parameter (e.g. <code>tcube</code>).</li>
        <li>Fix Aux axis positioning for 3D plots so that the numeric labels 
            don't get snipped off at top and bottom.</li>
        <li>Add a hack that allows LDAC FITS tables to be treated sensibly
            in auto-format-detection mode.</li>
        <li>Make VOTable handling more robust against unknown (illegal)
            datatypes.</li>
        <li>Add missing parameters <code>auxmax</code>, <code>auxmin</code>
            to plotting task documentation.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-3">Version 3.0-3 (14 April 2015)</dt>
<dd><p>
    <ul>
    <li>New System Command option for
        <ref id="location-syntax">input table syntax</ref>;
        you can now use "<code>&lt;syscmd</code>" or "<code>syscmd|</code>"
        to supply input byte streams from Un*x pipelines.</li>
    <li>Add new Kernel Density Estimate plot layer types
        <ref id="layer-kde">kde</ref>,
        <ref id="layer-knn">knn</ref> and
        <ref id="layer-densogram">densogram</ref>
        for <code>plot2plane</code>.</li>
    <li>More histogram normalisation options provided.
        Instead of just <code>true</code>/<code>false</code>,
        the <code>normalisation</code>
        parameter of the <ref id="layer-histogram">histogram</ref> layer
        now has the options
        <code>none</code>, <code>height</code>,
        <code>area</code> and <code>maximum</code>.
        This allows both the area normalisation introduced in v3.0-2,
        and the height normalisation used in earlier versions
        which it replaced.</li>
    <li>More histogram bar style options provided;
        the <ref id="layer-histogram">histogram</ref> <code>barform</code>
        parameter now provides the options
        <code>semi_filled</code> (the new default) and <code>semi_steps</code>.
        These give outlined partially transparent bars, which make it
        much easier to see what's going on in multi-dataset histograms.
        Note <code>semi_steps</code> does not currently export very nicely
        to PDF/EPS.
        Similar options are also available in the new KDE plots.</li>
    <li>Column data read in as unsigned bytes will now be written out as
        unsigned bytes where the output format permits; 
        previously they were forced to 16-bit signed integers.
        This affects FITS, VOTable and CDF I/O handlers.</li>
    <li>Add <code>count_rows()</code> method to JyStilts 
        <ref id="jytable">table</ref> objects, which for non-random tables
        may be much more efficient than <code>len()</code>.</li>
    <li>Be less strict about recognising colfits files
        (tolerate implicit TDIMn headers).</li>
    <li><code>taplint</code> is now aware of, and performs some checks
        related to, schema-level table metadata declared by TAP services.</li>
    <li>Work round FITS read bug that could cause problems for VOTables using
        inline FITS serialization, and possibly elsewhere.</li>
    <li>Fix bug that caused trouble when auto-ranging a plot
        with a single sky position.</li>
    </ul>
    </p></dd>

<dt id="v3.0-4">Version 3.0-4 (17 August 2015)</dt>
<dd><p>
    <dl>
    <dt>Bugfixes (some significant):</dt>
    <dd><p>
        <ul>
        <li>Fix a serious bug in processing of FITS bit vector
            (<code>TFORMn='rX'</code>) columns.
            Values read from these columns are presented as a
            <code>boolean[]</code> array.  In all previous versions of STIL
            the bits have appeared in that array in the wrong sequence
            (LSB..MSB per byte rather than the other way round).
            Apologies to anyone who may have got incorrect science results
            from this error in the past, and thanks to Paul Price for helping
            to diagnose it.</li>
        <li>Fix a less serious bug with <code>TFORMn='rX'</code> processing;
            attempting to read a single-element bit vector column
            (<code>TFORMn=1X</code> or <code>X</code>) previously
            resulted in an error making the file unreadable.
            Values read from such columns are now presented as
            Boolean scalars.</li>
        <li>Fix a VOTable reading bug relating to bit vector data
            (<code>datatype="bit"</code>) appearing in BINARY/BINARY2
            serializations.  This one was more obvious, it would usually
            generate an error when attempting to read the file.</li>
        <li>Fix serious bug in time conversion for CDF TIME_TT2000
            data types.</li>
        <li>Fix a bug in <code>votcopy</code> that converted columns from
            datatype <code>unsignedByte</code> to <code>short</code> when
            transforming.  Since v3.0-3 this is no longer necessary.
            In the case of converting to a binary serialization,
            since v3.0-3 this was causing it to generate unreadable VOTable
            output.</li>
        <li>Fix a bug in <code>votcopy</code> that failed to handle
            columns with datatype <code>bit</code>.  In the case of
            converting to a binary serialization, these were in all previous
            versions generating unreadable VOTable output.
            Now they convert them to columns with datatype
            <code>boolean</code> (not perfect, but better).</li>
        <li>Fix <code>skyvector</code> bug:
            <code>dlat</code> and <code>dlon</code>
            values were being used the wrong way round.</li>
        <li>Upgrade JEL to v2.0.2.
            Fixes problem with evaluating void-typed expressions,
            and possibly some other obscure bugs.</li>
        <li>Some taplint bug fixes.</li>
        </ul>
        </p></dd>
    <dt>Behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>Changes to the way that TAP service table/column name reports are
            interpreted (to conform to original intention of TAP standard).
            Taplint now checks that table/column names from TAP_SCHEMA
            and /tables
            endpoint are regular-or-delimited-identifiers, but no longer
            submits example queries using supplied column names wrapped in
            additional quotes.</li>
        <li>Modify the heuristics that determine whether the first row of
            a CSV file is a header.</li>
        </ul>
        </p></dd>
    <dt>Enhancements (mostly minor):</dt>
    <dd><p>
        <ul>
        <li>Added some functions to the
            <ref id="Arrays">Arrays</ref> class
            that return array-valued results from array-valued parameters:
            <code>add</code>, <code>subtract</code>,
            <code>multiply</code>, <code>divide</code>,
            <code>reciprocal</code>, <code>condition</code>.</li>
        <li>Improve error reporting in the face of non-VOTable 
            TAP error responses.
            In many cases this makes it much easier to see what's going wrong
            with a TAP query.</li>
        <li>As a diagnostic tool, when making TAP queries,
            a log message giving a roughly equivalent <code>curl(1)</code>
            command is now issued at the CONFIG level
            (visible using flags <code>-verbose -verbose</code>).</li>
        <li>New <code>taplint</code> parameter <code>maxtable</code>
            limits the number of tables tested in the stage that
            queries data from each individual table (<code>MDQ</code>).
            May be useful for very large services.</li>
        <li>New <code>tapquery</code> parameter <code>upvotformat</code>
            to determine what VOTable serialization variant is used to
            transmit uploaded tables to the TAP server.
            Previously uploads were always BINARY which ought to work,
            but the parameter now defaults to TABLEDATA,
            since some services (e.g. CADC)
            currently fail with binary uploads.</li>
        <li>Minor improvement to version reporting
            (reports java specification version,
             no longer issues warning for absent revision string).</li>
        <li>Update JCDF library to v1.1
            (minor changes to do with leap seconds).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-5">Version 3.0-5 (22 October 2015)</dt>
<dd><p>
    <ul>
    <li>Fix error reporting bug when a non-VOTable response is received
        from a TAP service.</li>
    <li>Upgrade to JCDF v1.2 - fixes a bug when reading large (multi-2Gb)
        CDF files.</li>
    <li>Added source code for an example basic GUI plot application,
        <code>uk.ac.starlink.ttools.example.BasicPlotGui</code>.</li>
    <li>The expression language has a new way of
        <ref id="jel-colref">referring to a column</ref>;
        if you use the form "<code>Object$</code><em>&lt;column-id&gt;</em>"
        you get the value as an Object not a primitive.
        This is a special-interest measure for user-defined
        functions that need to see null numeric values.</li>
    <li>Adjust GBIN input handler: 
        avoid descending into Class-typed members of gbin list objects, 
        and add logging for object-&gt;column translations.</li>
    </ul>
    </p></dd>

<dt id="v3.0-6">Version 3.0-6 (27 November 2015)</dt>
<dd><p>
    <dl>
    <dt>Crossmatching bug fix</dt>
    <dd><p>Fix a long-standing crossmatch bug
        relating to range restriction during pre-processing.
        This could have caused missed associations (but not false positives)
        near the edge of coverage regions when using per-row errors,
        if the scale of the errors differed
        (especially differed significantly) between the matched tables.
        It affected <code>matcher</code> values of
        <code>&lt;n&gt;-d_err</code>, <code>skyerr</code>,
        <code>2d_ellipse</code> and <code>skyellipse</code> only.
        Thanks to Grant Kennedy (IoA) for reporting this bug.
        </p></dd>
    <dt>Density plots</dt>
    <dd><p>Some more options for making weighted density plots have been added.
        Since v3.0 the Density shading mode has let you see the density
        of plotted points, but this lacked some features.
        Three new ways to do density plots are added; these all give you
        the option of weighting by an additional coordinate (like the Aux mode),
        choosing the combination method (mean, median, sum, max, ...),
        and displaying the quantitative value-colour mapping
        on the shared colour ramp (previously aux axis) beside the plot.
        The new density plots are:
        <ul>
        <li><ref id="shading-weighted">Weighted shading mode</ref>,
            using shaped marker kernels on the screen pixel grid,
            available for all plot types</li>
        <li><ref id="layer-skydensity">SkyDensity layer</ref>,
            using HEALPix bins on the celestial sphere, for the Sky plot</li>
        <li>Density layer
            (<em>later obsoleted by <ref id="layer-grid">Grid layer</ref></em>),
            using square N*N screen pixel bins, for the Plane plot</li>
        </ul>
        The details are somewhat experimental and may
        undergo some adjustments in future releases (feedback welcome).
        </p></dd>
    <dt>Colour maps</dt>
    <dd><p>There are various changes affecting selection and display of
        colour maps used for density and aux axis shading:
        <ul>
        <li>The default colour map for Aux mode, and other layers using
            the shared colour map, is no longer Rainbow!  It's Inferno.
            Rainbow colour maps are much hated by visualisation experts.
            Of course you can still choose Rainbow if you like.</li>
        <li>Add some new colour maps: 
            <em>Viridis</em>, <em>Inferno</em>, <em>Magma</em> and 
            <em>Plasma</em> from
            <webref url="http://bids.github.io/colormap/"
                    >Matplotlib 1.5</webref>,
            the <em>SRON</em> rainbow variant developed by
            <webref url="https://personal.sron.nl/~pault/"
                    >Paul Tol</webref>,
            some diverging maps 
            (<em>HotCold</em>, <em>RdBu</em>, <em>PiYG</em>, <em>BrBG</em>)
            and a qualitative constant chroma/luminance map 
            <em>HueCL</em>.</li>
        <li>The options for Density and Aux shading are now mostly the same
            as each other except where there's good reason to differ.
            Previously they were different in haphazard ways.</li>
        <li>An attempt is made to give the default form of each colour map
            a sensible name, without leading minus signs.</li>
        <li>Fix it so that the whole range of each map is distinguishable
            from white.  This is a good idea when you're plotting symbols
            on a white background, which is common in stilts.
            Perhaps there are cases it's not such a good idea;
            if you think so, complain and I may change it back.</li>
        <li>Try to fix it so that all the colour maps go in the same
            direction (light-&gt;dark) where applicable.</li>
        <li>Throw out a couple of particularly useless colour maps.</li>
        <li>Colour map ramp display is now different for non-absolute maps;
            their effect is shown on a selection of base colours,
            not just for one base colour.</li>
        </ul>
        </p></dd>
    <dt>Minor items</dt>
    <dd><p>
        <ul>
        <li>Try harder to identify epoch columns (suitable for time plot),
            in particular look for VOTable <code>xtype</code> of JD or MJD,
            and <code>units</code> of year.</li>
        <li>Add some functions to the
            <ref id="Tilings">Tilings</ref> class
            to do with solid angles
            (<code>healpixSqdeg</code>,
             <code>healpixSteradians</code>,
             <code>steradiansToSqdeg</code>,
             <code>sqdegToSteradians</code>,
             <code>SQDEG</code>).</li>
        <li>Fix plot bug; titles were painted in white
            for pixel output formats.</li>
        <li>Rationalise plot report logging.  Some more diagnostic information
            about plots is now logged at the INFO level
            (visible if topcat is run with the <code>-verbose</code> flag).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-7">Version 3.0-7 (10 June 2016)</dt>
<dd><p>
    <dl>
    <dt>Expression Language</dt>
    <dd><p>The JEL library underlying the expression language parser
        has been upgraded to v2.1.1 (thanks to Konstantin Metlov),
        now supporting variable-length argument lists among other things.
        This allows the following improvements:
        <ul>
        <li>New functions that support any number of arguments are provided:
            <code>array</code>, <code>intArray</code>, <code>stringArray</code>
            in class <ref id="Arrays">Arrays</ref>;
            <code>concat</code>, <code>join</code>
            in class <ref id="Strings">Strings</ref>;
            and
            <code>sum</code>, <code>mean</code>, <code>variance</code>,
            <code>stdev</code>, <code>min</code>, <code>max</code>,
            <code>median</code>, <code>countTrue</code>
            in new class
            <ref id="Lists">Lists</ref>.</li>
        <li>Some old lists of similarly-named functions with fixed numbers
            of arguments have been replaced by single functions that
            take an arbitrary number of arguments
            (e.g. <code>array(x1)</code>, <code>array(x1,x2)</code>, ...
                  <code>array(x1,x2,x3,x4,x5,x6,x7,x8)</code>
            replaced by <code>array(values...)</code>).</li>
        <li>The 2-argument <code>min</code>/<code>max</code> functions in class
            <code>Arithmetic</code> have been renamed
            <code>minNaN</code>/<code>maxNaN</code> to avoid confusion,
            but in most cases existing expressions involving min/max
            will work as before.</li>
        <li>Some functions that used to require string arguments will now
            auto-convert numeric types
            (e.g. <code>concat(toString(RA),";",toString(DEC))</code>
            can now be written <code>concat(RA,";",DEC)</code>).</li>
        <li>You can now implement
            <ref id="jelExtend">user-defined functions</ref> with variable
            numbers of arguments.</li>
        <li>Writing large (&gt;=2**31) literal integers used to fail with
            an inscrutable error message.
            Now the message tells you to append the
            "<code>L</code>" character.</li>
        </ul>
        </p></dd>
    <dt>Time plots</dt>
    <dd><p>The <ref id="plot2time"><code>plot2time</code></ref> command
        has been enhanced so that it can make <em>multi-zone</em> plots - 
        multiple plots stacked vertically that share the same horizontal
        (time) axis but have independent vertical axes.
        The time plot itself and this multi-zone feature are currently
        experimental; in future versions they may be improved or changed,
        and the multi-zone feature may be extended to other plot types.
        Some other changes and fixes have gone along with this:
        <ul>
        <li>A few API changes have been made to support multi-zone plots,
            including generalising the <code>NavigationListener</code>
            interface and rearranging some
            <code>PlotDisplay</code> and <code>AbstractPlot2Task</code>
            constructor/factory method arguments.
            For single-zone plots the changes are not very substantial.
            This only affects you if you are using the STILTS classes
            as a java plotting library.
            If that applies to you and you have trouble upgrading,
            I'm happy to provide assistance.</li>
        <li><code>plot2time</code> now supports shading modes
            (<code>shadingN</code> and associated parameters).</li>
        <li>The <ref id="layer-spectrogram">spectrogram</ref> layer
            now uses the (per-plot, or more precisely now per-zone)
            Aux colour map rather than a layer-specific colour map.
            This means that the colour ramp is displayed alongside the plot,
            but also that some parameters have been renamed
            (e.g. <code>auxmapZ</code> replaces <code>spectromapN</code>,
            where <code>Z</code> is an optional zone suffix
            and <code>N</code> is an optional layer suffix).</li>
        <li>The <code>function</code> layer type now works in the time plot,
            rather than throwing an error.
            However, it doesn't work very well, since the time coordinate
            is in unix seconds rather than something more user-friendly.</li>
        <li>Fixed a serious bug in ISO-8601 axis labelling.
            In some cases axis labels were being drawn at positions badly
            different from the correct position.</li>
        </ul>
        </p></dd>
    <dt>Miscellaneous enhancements and changes</dt>
    <dd><p>
        <ul>
        <li>This and subsequent releases target <strong>Java SE 6</strong>,
            so will no longer
            run under the (now very ancient) Java 5 runtime.</li>
        <li>Provide more careful documentation of licensing arrangements.
            The distributed LICENSE.txt file notes that the starjava code is
            LGPL, and documents licenses for each third-party dependency.</li>
        <li>Add <ref id="layer-fill">Fill</ref> layer type for Plane and Time
            plots.</li>
        <li><code>tapquery</code> and <code>tapresume</code> now use
            blocking HTTP requests rather than repeated polls
            to wait for asynchronous TAP job completion from
            services that declare themselves UWS 1.1 compliant.</li>
        <li>Add new parameters
            <code>executionduration</code> and <code>destruction</code> to
            <ref id="tapquery"><code>tapquery</code></ref> command.
            These let you request resource limit adjustments when submitting
            an asynchronous TAP job.</li>
        <li>Improve sky plot border painting.</li>
        <li>Clean up noisy Cubehelix colour map.</li>
        <li>New function <code>countTrue</code> in class
            <ref id="Arrays">Arrays</ref>.</li>
        <li>New stage <code>EXA</code> for <ref id="taplint">taplint</ref>
            checks TAP <code>/examples</code> endpoint.
            Note the details of the examples format are still under discussion,
            (this version targets WD-DALI-1.1-20160415 &amp;
            WD-TAP-1.1-20160428, somewhat informed by TAPNotes-2013-12-13),
            so the details may change in future.</li>
        <li><code>taplint</code> now validates ObsCore 1.1 where declared
            alongside ObsCore 1.0.
            Currently uses PR-ObsCore-v1.1-20160330.</li>
        <li>The <javadoc docset="&stiltsjavadocs;"
                         class="uk.ac.starlink.ttools.taplint."
                         >taplint API</javadoc>
            has changed slightly:
            the class that used to be <code>Reporter</code>
            is now called <code>TextOutputReporter</code>.
            If you are using taplint programmatically you may need to make
            small changes.  This results from some refactoring that makes
            it easier to customise taplint output.</li>
        <li>Replaced <code>opaque</code> config option with
            <code>transparency</code> for plane and sky density plots.</li>
        <li>Changed implementation of GIF exporter for plots,
            from Acme to ImageIO.
            Shouldn't be any noticable difference.
            Acme encoding dependency removed.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Fix bug in cumulative histogram calculation.</li>
        <li>Fix read failure for FITS files with non-blank TDIM for
            zero-length columns.</li>
        <li>Fix bugs that led to timezone-dependent results when
            reading ISO-8601 or decimal year time columns.</li>
        <li>Fix numeric field truncation bug in LaTeX table output.</li>
        <li>Fix some parameter handling errors in <code>coneskymatch</code>.
            </li>
        <li>Fix NullPointerException bug for disjoint regions in some cases in
            <code>tmatchn</code>.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.0-8">Version 3.0-8 (13 September 2016)</dt>
<dd><p>
    <ul>
    <li>New task <ref id="tskymap">tskymap</ref> lets you generate all-sky
        density maps (e.g. HEALPix) from an input table.</li>
    <li>New plot layer type <ref id="layer-healpix">healpix</ref>
        can plot all-sky maps from HEALPix map tables
        (e.g. as generated by <code>tskymap</code>).</li>
    <li>Add some HEALPix-related functions to the 
        <ref id="Tilings">Tilings</ref> class:
        conversions from pixel index to sky position and
        conversions between ring and nested schemes.</li>
    <li>Subrange-typed plot command parameters like
        <ref id="plot2plane-usage">plot2plane</ref>'s <code>auxclip</code>
        now have a default value of null.
        This uses a default clip, which avoids very light colours.
        Explicitly supplying a subrange value (e.g. "<code>0,1</code>")
        can now use the whole range; previously the very light colours
        were inaccessible.</li>
    <li>The GBIN input handler
        can now pick up more metadata from the classpath.
        For suitable tables, metadata included in datamodel classes 
        if present can be interrogated to provide table
        and column descriptions and UCDs.
        There are still some deficiencies of this functionality
        (no column order, utypes and units missing,
        large file "temp.xml" written to current directory)
        dependent on issues in the upstream Gaia libraries and ICD.</li>
    <li>Add parameters <code>tablesurl</code>, <code>examplesurl</code> etc
        to task <ref id="taplint"><code>taplint</code></ref>,
        so you can specify custom locations for different TAP endpoints
        rather than have them fixed at their default locations
        hanging off the base service URL.
        Ditto for the tasks <code>tapquery</code> and <code>tapskymatch</code>,
        though in these cases the extra parameters are
        (since currently rather special interest) largely undocumented.</li>
    <li>Fix <code>taplint</code> behaviour in absence of a /tables endpoint
        (404 now gives a Warning not an Error).</li>
    <li><code>taplint</code> ObsTAP validation updated to
        PR-ObsCore-v1.1-20160709.</li>
    <li>Fix bug that caused read failures for large (&gt;0.5Gb) FITS files
        outside of the current directory on 32-bit JVMs.
        This was a regression bug since v3.0-2.</li>
    <li>Fix long-standing bug that failed to release file descriptors
        when opening FITS tables (could cause an error if very many
        FITS files were opened).</li>
    </ul>
    </p></dd>

<dt id="v3.0-9">Version 3.0-9 (23 September 2016)</dt>
<dd><p>
    <ul>
    <li>Fix aux ranging bug in some plot types
        (SkyDensity, Healpix, Density, Weighted)
        that used an aux data range too small to show colour variations
        when <code>auxfunc=log</code>
        and negative data values were present.</li>
    <li>Improved performance (better memory use, faster) of some plot types
        (SkyDensity, Healpix, Density, Weighted).</li>
    <li>Fix JyStilts bug that could cause the same data to be used for
        different tables specified in a single plot.</li>
    </ul>
    </p></dd>

<dt id="v3.1">Version 3.1 (8 March 2017)</dt>
<dd><p>
    <dl>
    <dt>Plotting improvements</dt>
    <dd><p>
        <ul>
        <li>Improved documentation of plot
            <ref id="LayerType">layer types</ref> and
            <ref id="ShapeMode">shading modes</ref> in the user document -
            each option now has an example graphic
            and the text of the command that generates it.</li>
        <li>New layer type <ref id="layer-grid">grid</ref>,
            to plot a 2-d weighted histogram.
            This replaces the <code>density</code> layer,
            which has been withdrawn
            (<code>grid</code> can do all the same things and more,
            except specify bin size in screen pixels).</li>
        <li>New plot layer type <ref id="layer-quantile">quantile</ref>,
            which can (e.g.) plot median lines through noisy data.</li>
        <li>New plot layer type <ref id="layer-gaussian">gaussian</ref>,
            for Gaussian fits to histograms.</li>
        <li>Histogram-like layer types are now available from the
            <ref id="plot2time">plot2time</ref> command as well as
            from <code>plot2plane</code>,
            so you can now plot histograms with a temporal horizontal axis.</li>
        <li>New normalisation (scaling) option <code>unit</code> for
            histogram, KDE, and KNN plots.</li>
        <li>New normalisation (scaling) options <code>per_day</code> etc
            for histogram, KDE and KNN layers when used from the
            <code>plot2time</code> command.</li>
        <li>Colour names recognised by the <code>plot2*</code> command
            <code>*color</code> parameters
            now include the (140)
            <webref url="https://www.w3.org/TR/css-color-3/#svg-color"
                    >CSS-like</webref> colours alongside
            the dozen standard plotting colours.</li>
        <li>Plot commands <code>insets</code> parameter now lets you specify
            margins round plots using any combination of
            <code>&lt;top&gt;,&lt;left&gt;,&lt;bottom&gt;,&lt;right&gt;</code>,
            rather than requiring all values or none.</li>
        <li>Add new parameter <code>auxwidth</code> to plot2 commands,
            to control colour ramp lateral dimension in pixels.</li>
        <li>Modify <em>plan</em> caching arrangements for STILTS plots.
            This results in much better interactive performance
            (navigation etc) for some plot layer types plotted to the screen.
            There are also some minor API changes to a few of the plot2
            utility classes, which now provide more flexibility
            for caching.</li>
        <li>Various tweaks to the details of how plots are positioned on
            the screen or in output graphics files.</li>
        </ul>
        </p></dd>
    <dt>Miscellaneous enhancements</dt>
    <dd><p>
        <ul>
        <li>Fixed the match score (distance measure) for
            <ref id="CombinedMatchEngine">combined matchers</ref>.
            Previously, the score was a linear sum of the unscaled distances
            for the constituent matchers, which meant a <em>Best</em> match
            was pretty meaningless.
            Now, it adds scaled distances in quadrature,
            so Best matching should give you a somewhat sensible result.</li>
        <li>The <code><ref id="SkyMatchEngine-err">skyerr</ref></code>
            and <code><ref id="ErrorCartesianMatchEngine">nd_err</ref></code>
            matchers now report output separations as scaled (dimensionless)
            values rather than in physical units; this means the results are
            more comparable, so <em>Best</em> matches will make more sense.</li>
        <li><ref id="taplint">taplint</ref>
            can now optionally write its output in JSON
            format (see <code>format</code> parameter).</li>
        <li>Update <code>taplint</code> OBS stage for
            PR-ObsCore-v1.1-20160923.</li>
        <li>The <ref id="Maths">Maths</ref> function
            <code>hypot</code> now takes an arbitrary number of arguments
            (instead of exactly two).</li>
        <li>Add <code>blockmaxrec</code> parameter to
            <ref id="tapskymatch">tapskymatch</ref>,
            and improve warning text in case of result truncations.</li>
        <li>Improve <code>tapquery</code>/<code>tapresume</code> behaviour
            when monitoring async jobs; if the service goes down,
            the application will wait for it to come back rather than
            bailing out.</li>
        <li>Options for MOC output format
            (<code>mocfmt</code> parameter
            in <ref id="pixfoot">pixfoot</ref> command) have changed;
            option <code>ascii</code> is replaced by <code>json</code>.
            The old ASCII format was slightly broken JSON in any case.</li>
        <li>MOC library upgraded to v4.6 (from v3.3).  Improved MOC output.</li>
        <li>Update JCDF library to v1.2-2 (2017-01-01 leap second).</li>
        </ul>
        </p></dd>
    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Fix KDE/KNN plotting bug that could get scaling badly wrong
            for normalised cumulative plots.</li>
        <li>Fix some regression bugs relating to <code>plot2</code> command
            <code>insets</code> parameter, present since v3.0-6.</li>
        <li>Remove spurious padding from EPS graphics output.</li>
        <li>Fix small bug in <code>plot2plane</code>/<code>plot2time</code>
            label painting; horizontal axis label was sometimes off the
            bottom of the plot by a few pixels.</li>
        <li>Fix subpixel offset of colour ramp frame in PDF/PostScript graphics
            output.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.1-1">Version 3.1-1 (29 September 2017)</dt>
<dd><p>
    <dl>
    <dt>New functionality</dt>
    <dd><p>
        <ul>
        <li>New plot layer types
            <ref id="layer-xycorr">xycorr</ref> and
            <ref id="layer-skycorr">skycorr</ref> for error ellipses rotated
            as specified by Gaia-style correlation values.</li>
        <li>New plot layer type <ref id="layer-skygrid">skygrid</ref>
            can draw multiple sky system coordinate axis grids on sky plot.</li>
        <li>Colour map parameters
            (<code>auxmap</code>, <code>densemapN</code> etc)
            for the plotting commands
            will now accept custom colour maps that interpolate between
            a list of named colours,
            e.g. "<code>HotPink-yellow-SkyBlue</code>".</li>
        <li>New position angle calculation functions
            <ref id="CoordsDegrees"
                 ><code>posAngRadians</code></ref>
            and
            <ref id="CoordsRadians"
                 ><code>posAngDegrees</code></ref>
            added to expression language.</li>
        </ul>
        </p></dd>
    <dt>Table I/O changes</dt>
    <dd><p>
        <ul>
        <li>It is now possible to write and re-read tables with &gt;999
            columns to FITS or colfits format.  The new limit is 2^31 columns.
            This uses a non-standard convention; software that is not aware
            of the convention (e.g. CFITSIO or earlier STILTS versions)
            will only be able to use the first 998 columns of tables written
            in this way.</li>
        <li>For VOTable columns that reference <code>COOSYS</code> elements,
            the relevant information is now accessible as
            column auxiliary metadata
            (<code>CoosysSystem</code>, <code>CoosysEpoch</code>,
             <code>CoosysEquinox</code>)
            e.g. using the <ref id="meta"><code>meta</code></ref> filter.</li>
        <li>Any columns referencing <code>COOSYS</code> elements
            read from VOTable-based formats (VOTable or FITS-plus)
            will now be written out to VOTable-based formats with
            equivalent COOSYS references included.
            Currently not table parameters though.</li>
        <li>The default version for output VOTables is now VOTable 1.3.
            New output formats <code>votable-binary2-inline</code> and
            <code>votable-binary2-href</code> are now offered alongside
            the five previously available VOTable variants.</li>
        <li>Slight changes to the FITS-plus output handler VOTable formatting
            in the primary HDU; now uses default output VOTable version
            rather than VOTable 1.1.</li>
        <li>FITS keywords using the ESO
            <webref
                url="https://fits.gsfc.nasa.gov/registry/hierarch_keyword.html"
                >HIERARCH</webref>
            convention can now be read as table parameters
            rather than ignored when reading FITS tables.</li>
        </ul>
        </p></dd>
    <dt>Minor behaviour changes</dt>
    <dd><p>
        <ul>
        <li>The <code>charset</code> parameter of <code>votcopy</code>
            now defaults to UTF-8 rather than the system default.</li>
        <li>Replace parameter <code>autoscale</code> with <code>unit</code> in
            <ref id="layer-skyvector"><code>skyvector</code></ref> and
            <ref id="layer-skyellipse"><code>skyellipse</code></ref>
            plot layers.</li>
        <li>Change autoscaling defaults for plot layer types
            <ref id="layer-xyvector"><code>xyvector</code></ref>,
            <ref id="layer-xyzvector"><code>xyzvector</code></ref>,
            <ref id="layer-skyvector"><code>skyvector</code></ref>,
            <ref id="layer-xyellipse"><code>xyellipse</code></ref>,
            <ref id="layer-skyellipse"><code>skyellipse</code></ref>;
            autoscaling is now turned <em>off</em> by default.
            Apologies for this change to behaviour,
            but it's better for consistency with new layer types
            <ref id="layer-xycorr"><code>xycorr</code></ref> and
            <ref id="layer-skycorr"><code>skycorr</code></ref>,
            and presents less danger of misinterpreted plots.</li>
        <li>Taplint: downgrade type mismatch error <code>E_QTYP</code>
            to warning <code>W_QTYP</code> in view of TAP-1.0 Erratum #3.</li>
        <li>The <code>COOSYS</code> element is now passed without a deprecation
            warning in VOTable 1.3 documents by
            the <code>taplint</code> and <code>votlint</code> validators,
            in view of VOTable-1.3 Erratum #1.
            The warning is still issued for VOTable 1.2 documents,
            but the text is toned down.</li>
        <li>Taplint no longer issues a warning about DataModel references
            that do not match the <code>vr:IdentifierURI</code> type;
            this follows TAPRegExt Erratum #1.</li>
        <li>Modify taplint Examples document @vocab attribute testing in the
            light of DALI 1.1.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Update PixTools (HEALPix) library to 2017-09-06 version
            (https://github.com/kuropat/eag-HEALPix,
             <code>447a7be073876dba32</code>).
            This fixes a bug in
            <ref id="Tilings"
                 ><code>healpixRingIndex</code></ref> that could give
            the wrong value for small values of longitude near zero
            in the equatorial region.
            It seems possible that this might have led to very infrequent
            missed associations when crossmatching in these regions,
            but tests appear to indicate that no such errors would
            actually have resulted.</li>
        <li>Long fields (&gt;10240 characters) in output CSV files
            are no longer truncated.</li>
        <li>Fix misfeature in
            <ref id="layer-skyvector">skyvector</ref> and
            <ref id="layer-skyellipse">skyellipse</ref> plot layers;
            these now preserve orientation on the sky even when
            <code>viewsys</code> differs from <code>datasys</code>.
            Previously the
            <code>dlat</code>/<code>dlon</code>/<code>posang</code>
            parameters were always interpreted in
            the view, rather than the data, sky coordinate system.</li>
        <li>Fix bug in plot title placement.</li>
        <li>Fix filename generation error for plotting command
            animation output (wrong number of digits if frame count
            was an exact power of 10).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.1-2">Version 3.1-2 (7 November 2017)</dt>
<dd><p>
    <dl>
    <dt>New expression language functions</dt>
    <dd><p>
        <ul>
        <li>New functions
            <ref id="Strings"
                 ><code>desigTo*</code></ref>
            for extracting positions from IAU-style object designations
            (like <code>2MASS J04355524+1630331</code>) - use with care.</li>
        <li>New functions
            <ref id="CoordsDegrees"><code>polarDistanceDegrees</code></ref> and
            <ref id="CoordsRadians"><code>polarDistanceRadians</code></ref>;
            these calculate the distance in 3d space between two
            positions specified in spherical polar coordinates.</li>
        <li>New functions
            <ref id="Arithmetic"><code>phase</code></ref>
            to help with phase folding given a known period.
            A new modulus function
            <ref id="Arithmetic"><code>mod</code></ref>,
            whose output is always positive
            (unlike the <code>%</code> operator) is also added.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes and workarounds</dt>
    <dd><p>
        <ul>
        <li>Upgrade JEL to v2.1.2: now you can use functions in the expression
            language that have the same name as table columns.</li>
        <li>Add fix to discard on read any VOTable/FITS-plus table parameters
            with names "<code>uk.ac.starlink.topcat.plot2.TopcatLayer*</code>".
            These useless items were added in potentially large numbers when
            saving plotted tables from TOPCAT v4.5 (TOPCAT bug).
            A null <code>tpipe</code> operation at this version will therefore
            purge these items.</li>
        <li>Fix plot bug that sometimes caused error bars to come out
            very small when exporting to a graphics file.
            This was a regression bug (present in v3.1-1 but not v3.1).</li>
        <li>Fix another tiny error bar plotting bug too.</li>
        <li>Small taplint update; required UCD for
            <code>obs_publisher_did</code>/<code>publisher_id</code> changed
            to match REC-ObsCore-1.1 (even though it's an illegal UCD1+).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.1-3">Version 3.1-3 (24 April 2018)</dt>
<dd><p>
    <dl>
    <dt>New Functionality</dt>
    <dd><p>
        <ul>
        <li>New DataLink validator task
            <ref id="datalinklint"><code>datalinklint</code></ref>.</li>
        <li>Add class <ref id="Gaia">Gaia</ref> to the expression language.
            This contains functions for estimating distances from parallaxes
            and propagating astrometric parameters and errors to
            different epochs.  The functions are not specific
            to data from the Gaia astrometry satellite,
            but are presented in a form convenient for use
            with the Gaia DR2 source catalogue.</li>
        <li>New array manipulation functions
            <code>slice</code> and <code>pick</code> added to class
            <ref id="Arrays">Arrays</ref>.</li>
        <li>New utility function <code>square</code> added to class
            <ref id="Maths">Maths</ref>.</li>
        <li>The <ref id="addcol"><code>addcol</code></ref>
            and <ref id="colmeta"><code>colmeta</code></ref> filters
            now accept <code>-shape</code> and <code>-elsize</code> flags
            for defining array and string shape/extent column metadata.</li>
        <li>DataLink-style service descriptor RESOURCEs in input VOTables
             (and FITS-plus tables) now appear as table parameters.</li>
        </ul>
        </p></dd>
    <dt>Plotting Enhancements</dt>
    <dd><p>
        <ul>
        <li>Various improvements to the <ref id="layer-contour">Contour</ref>
            plotter.
            You can now contour quantities weighted by a given coordinate,
            rather than just point density.
            The smoothing from weighted point samples to the grid being
            contoured can be performed using sum, mean and
            other combination methods,
            it now uses a Gaussian kernel rather than a square top hat,
            and performance is considerably improved especially at large
            smoothing widths.
            The contour levels used are now <em>reported</em>,
            so can be seen by specifying the <code>-verbose</code> flag.
            The contours are now plotted correctly right up to the edge of
            the visible plot, and various bugs have been fixed.</li>
        <li>The <ref id="layer-histogram">histogram</ref> plotter
            now has a <code>combine</code> parameter
            (<code>sum</code>,
             <code>mean</code>,
             <code>median</code>,
             <code>min</code>,
             <code>max</code>,
             <code>stdev</code>, etc)
            for weighted histograms.
            This means that you can plot e.g. the mean value of a given quantity
            per interval on the X axis rather than just the summed quantity.
            A corresponding (though somewhat less well-defined) option is
            also provided for the
            <ref id="layer-kde">kde</ref> and
            <ref id="layer-densogram">densogram</ref> plotters.</li>
        <li>The <code>combine</code> parameter that configures how
            values are binned in various histogram-like plots
            (<ref id="layer-skydensity">skydensity</ref>,
             <ref id="layer-healpix">healpix</ref>)
             <ref id="layer-grid">grid</ref>,
             and since this version also
             <ref id="layer-contour">contour</ref>,
             <ref id="layer-histogram">histogram</ref>,
             <ref id="layer-kde">kde</ref>)
            now has two new options,
            <code>sum-per-unit</code> and <code>count-per-unit</code>.
            These work like the existing <code>sum</code> and <code>count</code>
            options, but scale the combined values by the relevant unit
            (e.g. X axis unit or solid angle).
            Where these units are physical, a <code>perunit</code>
            parameter is also provided for scaling in convenient units:
            second, day, year etc for time
            (<ref id="plot2time">plot2time</ref>), and
            square degree, arcminute, arcecond etc for solid angle
            (<ref id="layer-skydensity">skydensity</ref>,
             <ref id="layer-healpix">healpix</ref>).
            In some cases, the <code>combine</code> default values
            have changed.</li>
        <li>The same new <code>combine</code>/<code>perunit</code> options
            are also added to the <ref id="tskymap">tskymap</ref> command.</li>
        <li>The <ref id="layer-linearfit">linearfit</ref> layer type
            is now available for
            <ref id="plot2time"><code>plot2time</code></ref>
            as well as <code>plot2plane</code>.</li>
        </ul>
        </p></dd>
    <dt>Minor behaviour changes</dt>
    <dd><p>
        <ul>
        <li><code>votlint</code> no longer considers an integer- or array-typed
            <code>PARAM</code> element with attribute <code>value=""</code>
            to be an error.</li>
        <li><code>votlint</code> no longer Warns about
            <code>PARAM/@ref</code>-&gt;<code>FIELD/@ID</code> references,
            since this usage pattern is now found in
            DataLink Service Descriptors.</li>
            <li>The IPAC table reader now matches data type specifications
                case-insensitively.</li>
        <li>Improvements in documentation of the expression language functions:
            in the <ref id="staticMethods">Function documentation</ref> section
            classes are now listed in alphabetical order,
            and examples are included in some cases.
            Some readability improvements have also been made in the function
            browser displayed by the <ref id="funcs"><code>funcs</code></ref>
            command.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes</dt>
    <dd><p>
        <ul>
        <li><code>taplint</code>'s VOTable validation now includes
            XML schema/DTD
            validation as well as the custom VOTable checks.
            It was always supposed to do this, but it seems that it didn't,
            at least in recent versions.</li>
        <li>Update JCDF to v1.2-3; fixes some CDF reading bugs.</li>
        <li>Prevent <code>coneskymatch</code> from sending cone search queries
            with <code>sr=NaN</code>.</li>
        <li>Minor fix in <code>taplint</code> OBS stage;
            ObsCore v1.1 <code>s_region</code> UCD is
            <code>pos.outline;obs.field</code>, not
            <code>phys.outline;obs.field</code>
            (a relic from PR-ObsCore-1.1-20161004).</li>
        </ul>
        </p></dd>
    </dl>
    </p>
    </dd>

<dt id="v3.1-4">Version 3.1-4 (18 May 2018)</dt>
<dd><p>Bugfixes and minor enhancements:
    <ul>
    <li>Add new option <code>delete=now</code> to
        <ref id="tapresume"><code>tapresume</code></ref>,
        which lets you delete a TAP async job.
        Also add a message to <ref id="tapquery"><code>tapquery</code></ref>
        inviting the user to delete interrupted jobs in this way.</li>
    <li>Upgrade Gr&#x00e9;gory Mantelet's ADQL parsing library to v1.4.</li>
    <li>VOTable output tweaked: single quotes now used for attribute
        values if attribute contains double quotes.</li>
    <li>Fix bug in <code>tapquery</code> and <code>tapresume</code> that
        caused a result read failure for result VOTables longer than 2Gb.</li>
    <li>Fix range bug; caused plot failure when plotting very
        large values with no variation
        (e.g. gaia_source solution_id).</li>
    </ul>
    </p>
    </dd>

<dt id="v3.1-5">Version 3.1-5 (2 November 2018)</dt>
<dd><p>
    <dl>
    <dt>Plotting:</dt>
    <dd><p>
        <ul>
        <li>New <ref id="layer-spheregrid">spheregrid</ref> plot layer,
            for plotting a spherical net around the origin,
            added to <code>plot2cube</code> and <code>plot2sphere</code>.</li>
        <li>New <ref id="layer-line3d">line3d</ref> plot layer,
            for plotting lines joining points in 3d,
            added to <code>plot2cube</code> and <code>plot2sphere</code>.</li>
        <li>New <code>sortaxis</code> parameter,
            for joining out-of-sequence points,
            added to <ref id="layer-line">line</ref> plot layer.</li>
        <li>Replace <ref id="layer-skygrid">skygrid</ref> plot layer
            parameter <code>crowd</code> with
            two parameters, <code>loncrowd</code> and <code>latcrowd</code>,
            so you can control grid line spacing for meridians
            and parallels independently.  Note this is a backwardly-incompatible
            change, so could break existing scripts.
            This change may have introduced slight changes to sky axis
            grid line spacing at low crowding levels.</li>
        <li><ref id="layer-contour">Contour</ref> level calculations improved;
            in some cases this previously didn't work well
            at low point density, resulting in missing contours.
            The meaning of the <code>zero</code> parameter has also changed
            slightly, it now defaults to 1 and is not phase folded.</li>
        <li>Sphere plot now centers axes on zero
            if the center would otherwise be near zero.</li>
        <li>Address long-standing plot auto-ranging issue;
            when auto-ranging resulted in the bottom X limit or
            right-hand Y limit being exactly equal to zero,
            the corresponding zero-valued results were not plotted.
            Auto-ranging has been slightly adjusted to avoid that.
            This results in pixel-level changes to plot appearance
            in some cases; these can be avoided by explicitly setting
            <code>xmax</code>/<code>ymin</code> etc.</li>
        </ul>
        </p></dd>
    <dt>Expression Language:</dt>
    <dd><p>
        <ul>
        <li>New functions in class
            <ref id="Gaia">Gaia</ref>:
            <code>polarXYZ</code>, <code>astromXYZ</code>,
            <code>astromUVW</code>,
            <code>icrsToGal</code>, <code>galToIcrs</code>,
            <code>icrsToEcl</code>, <code>eclToIcrs</code>.
            These can calculate Cartesian position and velocity components
            from (e.g. Gaia) astrometric parameters.</li>
        <li>New <ref id="Shapes">Shapes</ref>
            class added for working with polygons in the X,Y plane;
            has functions
            <code>isInside</code> and <code>polyLine</code>.</li>
        <li>Rearrange expression language documentation slightly:
            there is a new section listing
            <ref id="jel-special">Special Tokens</ref>.
            This includes "<code>$random</code>", which was previously
            undocumented and named "<code>RANDOM</code>".</li>
        <li>New functions <code>urlEncode</code> and <code>urlDecode</code>
            added to class
            <ref id="Strings">Strings</ref>.</li>
        <li>New convenience function <code>exp10</code> in class
            <ref id="Maths">Maths</ref>.</li>
        </ul>
        </p></dd>
    <dt>Validators:</dt>
    <dd><p>
        <ul>
        <li>Improvements to <code>taplint</code> UWS stage:
            now validates job documents
            against UWS schema, tests <code>uws:job/@version</code>
            attribute, and does improved and version-sensitive
            (UWS v1.0/v1.1) validation of UWS timestamps.
            Also does more complete deletion of submitted jobs.</li>
        <li><code>taplint</code> schema validation now takes account of
            <webref
               url="https://wiki.ivoa.net/twiki/bin/view/IVOA/VOTable-1_3-Err-2"
                    >VOTable 1.3 Erratum #2</webref> -
            <code>FIELD/@precision</code> attribute values
            "<code>F0</code>"/"<code>E0</code>" are now permitted.</li>
        <li>The VOTable validator in <code>votlint</code>
            and <code>taplint</code> no longer complains about multiple
            <code>INFO</code> elements in the same scope sharing the
            same <code>name</code> attribute.</li>
        <li>Minor improvements to <code>datalinklint</code>
            error reporting.</li>
        </ul>
        </p></dd>
    <dt>Miscellaneous Enhancements:</dt>
    <dd><p>
        <ul>
        <li>New <ref id="collapsecols"><code>collapsecols</code></ref>
            filter added,
            to convert a run of adjacent scalar columns into an
            array column (the opposite of <code>explodecols</code>).</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>GBIN read fix to work around changed behaviour in recent
            GaiaTools (versions 19.4.*, &gt;=20.1.0 and &gt;=21.0.0)
            that caused GBIN table reading to fail.</li>
        <li>Fix some additional commands
            (<code>votcopy</code>, <code>votlint</code>,
             <code>datalinklint</code>)
            so they follow HTTP 3xx redirects automatically in more cases
            when given a URL for input.
            Most STILTS commands already do this.</li>
        <li>Introduce a couple of measures to reduce the likelihood
            of unintentional service overload from
            <ref id="coneskymatch"><code>coneskymatch</code></ref> and friends:
            add a progressively increasing delay for <code>erract=retry*</code>
            error handling modes,
            and decrease the soft maximum for the <code>parallel</code>
            parameter to 5 (from 10).</li>

        <li>Slight improvements to the
            <ref id="jdbcConfig">JDBC Configuration</ref>
            section of this manual.</li>
        <li>Small change to MOC handling that might possibly avoid some
            network-related performance issues.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix <code>function</code> plot layer so that NaN values are
            omitted rather than interpolated.</li>
        <li>The <code>stilts</code> startup script now correctly follows
            symlinks on some OSes where it didn't work before
            (including OSX).  It also looks for jar files to use in
            slightly different places on OS X.</li>
        </ul>
        </p></dd>
    </dl>
    </p>
    </dd>

<dt id="v3.1-6">Version 3.1-6 (9 May 2019)</dt>
<dd><p>
    <dl>
    <dt>Plotting enhancements</dt>
    <dd><p>
        <ul>
        <li>Introduced new colour scaling options <code>histogram</code>
            and <code>histolog</code> for the parameters
            <code>auxfunc</code>, <code>densefunc</code> etc
            in <ref id="plot2plane"><code>plot2plane</code></ref>
            and the other plotting commands.
            This can make it much easier to see structure in quantities 
            that do not vary smoothly over their min-max range.</li>
        <li>New plot layer types
            <ref id="layer-poly4">poly4</ref>,
            <ref id="layer-mark4">mark4</ref>,
            <ref id="layer-polygon">polygon</ref>
            introduced for drawing outline or filled quadrilaterals and
            other polygons in all plot types.</li>
        <li>The <ref id="layer-line"><code>line</code></ref> plot layer
            now has an <code>aux</code> parameter that can vary the colour
            of the line along its length according to some
            third quantity.</li>
        <li><ref id="plot2sky"><code>plot2sky</code></ref> now by default
            draws a small scale bar at the bottom left corner of the plot,
            indicating the scale in degrees, minutes or seconds.
            It can be switched off using the new <code>scalebar</code>
            parameter.</li>
        <li>New option <code>Car0</code> for the
            <ref id="plot2sky"><code>plot2sky</code></ref>
            <code>projection</code> parameter;
            this is like <code>Car</code> (Plate Carr&#x00e9;e)
            but has longitude=0 at the left/right edge
            rather than the center of the plot.</li>
        <li>The <ref id="layer-healpix">healpix</ref> plot layer can now
            plot HEALPix levels up to 20 (previously the maximum was 13).
            Also, autoranging now works, so if a plot is made of a HEALPix
            file covering part of the sky, the sky view is centered on it
            in the same way as for marker plots.
            Memory management is improved for fairly large maps.</li>
        </ul>
        </p></dd>
    <dt>HEALPix-FITS support</dt>
    <dd><p>Various changes introduced to support the semi-standard
        <webref url="&URL.HEALPIX_FITS;">HEALPix-FITS</webref>
        serialization convention.
        Available information about HEALPix encoding
        (level, index column, ordering scheme, coord sys)
        can now be stored in custom table parameters
        (of the form <code>STIL_HPX_*</code>),
        and is used by FITS output handlers to insert
        the relevant FITS headers.
        The existing FITS handlers do this where it's not disruptive,
        and the new <ref id="outFormats">fits-healpix</ref>
        output handler will additionally move and rename columns
        if required.
        This metadata is round-tripped by FITS and VOTable I/O handlers.
        It is added automatically by the <ref id="tskymap">tskymap</ref>
        command, and can be manipulated by hand using
        the new <ref id="healpixmeta">healpixmeta</ref> filter.
        FITS support is not perfect: the <code>BAD_DATA</code>
        FITS keyword is ignored, and the 1024-element array-valued
        column variant is not understood.
        </p></dd>
    <dt>VOTable 1.4 support</dt>
    <dd><p>Support has been introduced for version 1.4
        of the VOTable format and its new <code>TIMESYS</code> element.
        <ul>
        <li><ref id="votlint"><code>votlint</code></ref>
            (and hence <ref id="taplint"><code>taplint</code></ref>)
            now supports Version 1.4 VOTables:
            the 1.4 schema is used for XSD validation,
            and the TIMESYS element is checked
            for attribute content and suitable referencing.</li>
        <li>For VOTable columns that reference TIMESYS elements,
            the relevant information is now accessible as
            column auxiliary metadata
            (<code>TimesysTimeorigin</code>,
             <code>TimesysTimescale</code>,
             <code>TimesysRefposition</code>)
            e.g. using the
            <ref id="meta"><code>meta</code></ref> filter.</li>
        <li>Any columns referencing TIMESYS elements
            read from VOTable-based formats (VOTable or FITS-plus)
            can now be written out to VOTable-based formats with
            equivalent TIMESYS references included,
            so TIMESYS round-tripping for columns works;
            however this will only be done if the VOTable output
            format is set to version 1.4.
            By default (at least as long as 1.4 is not finalised)
            the output version is 1.3.
            To enable this TIMESYS output, set the
            <ref id="sysProperties">system property</ref>
            <code>-Dvotable.version=1.4</code>.
            Currently this TIMESYS output works only for table columns
            (FIELDs) not parameters (PARAMs).</li>
        <li>The <code>timeoffset</code> attribute of a TIMESYS element
            referenced by a VOTable column is used to make sense of
            column data when interpreting it as an absolute time.
            Currently, the only use of this is in
            <ref id="plot2time">time plots</ref>.</li>
        </ul>
        This VOTable 1.4 support has resulted in some minor related
        behaviour changes:
        <ul>
        <li><code>FIELD/@ref</code> attributes are no longer imported
            as "<code>VOTable ref</code>" column aux metadata items,
            since they often interfere with TIMESYS references.
            Doing this was probably always a bad idea since the referencing
            is not kept track of within the application, so withdrawing
            this functionality makes sense, but beware that it might
            change or break some existing behaviour.</li>
        <li><code>votlint</code> modifications above may now
            interrogate external vocabulary resources during validation,
            meaning that external network connections may be made during
            validation, which didn't happen in previous versions.</li>
        </ul>
        </p></dd>
    <dt>Other enhancements</dt>
    <dd><p>
        <ul>
        <li>New functions <code>indexOf</code> in class
            <ref id="Arrays">Arrays</ref>
            to find position of a given value in an array.</li>
        <li>New functions <code>parseDoubles</code> and <code>parseInts</code>
            in class
            <ref id="Conversions">Conversions</ref>
            for extracting array values from strings (experimental).</li>
        <li>Fix expression compilation so that
            <code>$ID</code> column references
            referring to nonexistent columns are rejected at compile time 
            rather than causing trouble during evaluation.</li>
        <li>The validator tasks
            <ref id="taplint"><code>taplint</code></ref> and
            <ref id="datalinklint"><code>datalinklint</code></ref> now
            insert the token "<code>(IVOA-validate)</code>" into the
            <code>User-Agent</code> header of all HTTP requests they make,
            for convenience of services that want to identify validators.
            This is a convention discussed within the IVOA Operations IG.</li>
        </ul>
        </p></dd>
    <dt>Bug and misfeature fixes</dt>
    <dd><p>
        <ul>
        <li>Improve colour ramp quantisation
            (plot parameters <code>auxquant</code> etc);
            the full color range is now included.</li>
        <li>Use period not comma as decimal separator for non-sexagesimal
            <code>plot2sky</code> axis labels regardless of Locale;
            also avoid trailing comma sometimes erroneously present.</li>
        <li>Fix bug/misfeature in CDF table parameter construction:
            CDF global attributes were ignored
            (with a "WARNING: Omitting complicated global attribute" message)
            if they contained any null entries.
            Now such entries are just ignored and the table parameter is
            constructed from the global attribute using the
            non-null entries.</li>
        <li>Fix some bugs relating to plotting values close to the limits
            of the double precision range.</li>
        <li>Fix a problem with the <ref id="layer-link2">link2</ref>
            plot layers on sky plots with Aitoff/Car projections
            that caused short lines that should span the antimeridian
            to appear as long lines crossing the whole sky.
            Such links are now just not drawn.</li>
        <li>Fix misleading error message about <code>seq</code> parameter
            for underconfigured plot layers in <code>plot2*</code>
            commands.</li>
        </ul>
        </p></dd>
    </dl>
    </p>
    </dd>

<dt id="v3.2">Version 3.2 (18 November 2019)</dt>
<dd><p>
    <dl>
    <dt>Runtime environment: Java 8</dt>
    <dd><p>From this release, <strong>STILTS requires Java 8</strong>
        (a.k.a. Java 1.8) or greater to run,
        rather than Java 6 as for previous releases.
        Java 8 has been around since 2014, so it should be available 
        on all but very ancient platforms.
        If execution fails with a
        <code>java.lang.UnsupportedClassVersionError</code>
        then you need to upgrade.
        </p></dd>
    <dt>New or enhanced functionality:</dt>
    <dd><p>
        <ul>
        <li>New <ref id="cone"><code>cone</code></ref> command to execute
            simple spatial DAL queries (Cone Search, SIA, SSA).</li>
        <li>Most plot layer types, though not the intial data preparation,
            will now run in parallel for large datasets.
            When using one of the <ref id="plot2"><code>plot2*</code></ref>
            commands interactively (<code>omode=swing</code>), this should make
            interacting with slow plots faster on multi-core machines.
            (In rare cases this multi-threading might cause problems with
            memory usage; it can be effectively turned off if required
            by using the <ref id="sysProperties">system property</ref>
            <code>java.util.concurrent.ForkJoinPool.common.parallelism</code>.)
            </li>
        <li>New class <ref id="URLs">URLs</ref>
            contains expression language utility functions
            for constructing certain service URLs:
            <code>hips2fitsUrl</code>, <code>bibcodeUrl</code>,
            <code>doiUrl</code>, <code>arxivUrl</code>,
            <code>simbadUrl</code> and <code>nedUrl</code>.
            Existing functions <code>urlEncode</code> and
            <code>urlDecode</code> have been moved to <code>URLs</code>
            from class
            <ref id="Strings">Strings</ref>.</li>
        <li>New functions
            <code>parseBigInteger</code> and <code>parseBigDecimal</code>
            in class
            <ref id="Conversions">Conversions</ref>.</li>
        <li>Add new parameter <code>usepos</code> to
            <ref id="layer-polygon"><code>polygon</code></ref> plot layer,
            to toggle inclusion of reference position
            in polygon vertex list.</li>
        <li>Add parameter <code>interface</code> added to TAP query tasks
            <code>tapquery</code>, <code>tapskymatch</code> and
            <code>taplint</code>.  This can control how TAP version
            is determined.</li>
        <li>Modified behaviour for offset (e.g. unsigned) longs in FITS files.
            64-bit integer columns (<code>TFORMn='K'</code>)
            with non-zero integer offsets
            (<code>TSCALn=1</code>, <code>TZEROn&lt;&gt;0</code>)
            are now represented internally as Strings;
            previously they were represented as Long integers,
            but values out of the possible range appeared as null
            (with a warning written through the logging system).
            Such columns are most commonly seen representing
            unsigned long values.
            If written back out to FITS, the offset long value will be
            reinstated, but other output formats cannot encode
            unsigned longs, so they will stay as strings.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>VOTable output now writes VOTable version 1.4 by default
            (was 1.3).</li>
        <li>Modified plot legend display so that small markers are displayed
            a bit bigger in the legend than on the plot for readability.</li>
        <li>Add new colour map Cividis.</li>
        <li>Permit FITS and VOTable files with zero-length string columns.
            Previously all-null or zero-length string columns were 
            sometimes forced to single-character values.</li>
        <li>Update mapped file unmapping implementation to work
            for java9+.</li>
        <li>Updated <code>taplint</code> OBS stage according to
            ObsCore-1.1 Erratum #1 (corrected UCDs).
            No code changes required for ObsCore-1.1 Erratum #2.</li>
        <li>More taplint tests added for TAP 1.1-style capabilities
            document.</li>
        <li>Taplint now recognises ADQL 2.1 language features
            (as listed in PR-ADQL-2.1-20180112),
            avoiding some <code>E_KEYX</code> errors.</li>
        <li>VOResource validation in <code>taplint</code> now uses
            VOResource v1.1 not v1.0.
            Erratum #1 (multiple security methods) has been applied.</li>
        <li>Some <code>taplint</code> enhancements: add some new general tests,
            and some only used when testing TAP 1.1 services.</li>
        <li>Adopt VOTable-1.3 Erratum #3 in
            <ref id="votlint"><code>votlint</code></ref>;
            <code>arraysize="1"</code> now provokes a Warning.</li>
        <li>Updates to <ref id="meta"><code>meta</code></ref>
            filter documentation.</li>
        <li>Minor changes to behaviour when querying TAP 1.1 services
            (REQUEST parameter is omitted).</li>
        <li>JSAMP to version 1.3.7.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix regression issues introduced at v3.1-6 relating to
            <ref id="pixsample"><code>pixsample</code></ref> command;
            it now reads HEALPix metadata from headers correctly again.</li>
        <li>Avoid sometimes losing precision when reading ASCII/CSV values
            in the range +/-(1e-45..1e-38).</li>
        <li>Fix plot axis ranging bug: padding was not always applied 
            properly for logarithmic axes.</li>
        <li>JEL bug fix update, to avoid unwanted debugging output for
            String function null returns.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.2-1">Version 3.2-1 (5 June 2020)</dt>
<dd><p>
    <dl>
    <dt>File Formats:</dt>
    <dd><p>
        <ul>
        <li>The <label>ECSV</label> (Enhanced Character Separated Values)
            storage format is now supported for
            <ref id="inFormats">input</ref> and
            <ref id="outFormats">output</ref>.</li>
        <li>The <label>Feather</label>
            storage format is now supported for
            <ref id="inFormats">input</ref> and
            <ref id="outFormats">output</ref>.</li>
        </ul>
        </p></dd>
    <dt>Performance:</dt>
    <dd><p>A number of implementation changes have been made
        which may improve performance, particularly for crossmatching
        (typical improvements for large sky crossmatches are a factor of 2,
        though YMMV).  These should have no effect on the results,
        but if anybody notices crossmatching behaviour which is
        changed since previous versions or otherwise suspicious,
        please report it.
        <ul>
        <li>The HEALPix implementation has been replaced;
            all HEALPix manipulation is now done using the excellent
            <webref url="https://github.com/cds-astro/cds-healpix-java"
                    >cds-healpix-java</webref> library
            written by Fran&#x00e7;ois-Xavier Pineau from CDS,
            which speeds up sky crossmatching considerably.
            Many thanks to Fran&#x00e7;ois-Xavier and to CDS
            for providing this library and for assistance with its use;
            thanks also to Nikolay Kuropatkin from FermiLab whose
            <webref url="https://home.fnal.gov/~kuropat/HEALPIX/PixTools.html"
                    >PixTools</webref> library
            served this purpose in STILTS up till now.</li>
        <li>Rows are now binned during crossmatches
            using a HashSet rather than a TreeSet.</li>
        <li>Evaluations of the <code>arcsin</code> function in Sky matches
            now use the (Apache via cds-healpix) FastMath implementation
            rather than the standard J2SE version.</li>
        <li>Performance is improved when reading long String values
            from FITS files.</li>
        </ul>
        </p></dd>
    <dt>Plot commands:</dt>
    <dd><p>
        <ul>
        <li>The new layer types
            <ref id="layer-area"><code>area</code></ref>,
            <ref id="layer-central"><code>central</code></ref> and
            <ref id="layer-arealabel"><code>arealabel</code></ref>
            can be used for plotting region data supplied as
            area coordinates in the form of
            STC-S (e.g. from ObsCore/EPN-TAP <code>s_region</code>),
            DALI polygon/circle/point, or (ASCII) MOC columns.
            These layers are available from the
            <ref id="plot2plane"><code>plot2plane</code></ref>,
            <ref id="plot2sky"><code>plot2sky</code></ref> and
            <ref id="plot2sphere"><code>plot2sphere</code></ref> commands.</li>
        <li>The <ref id="plot2time"><code>plot2time</code></ref> command
            has been improved,
            and is no longer considered <em>experimental</em>.
            A new <code>ttypeN</code> parameter is added,
            which allows you to explicitly define (as MJD, JD, ISO-8601 etc)
            how input values are mapped to time. 
            If no such mapping is specified or can be guessed from
            the input metadata, time values are now interpreted by default
            as MJD rather than (as previously) Unix seconds.</li>
        <li>Provide more options for the <label>shapeN</label> parameter
            of the plot <ref id="layer-mark"><code>mark</code></ref> layer type:
            <code>fat_circle</code>, <code>fat_cross</code> etc
            using thicker lines.</li>
        <li>Add line thickness parameter <code>thick</code> to
            <ref id="layer-contour"><code>contour</code></ref> plot.</li>
        <li>Plot <code>storage</code> parameter has new options
            <code>disk</code>, <code>policy</code> and
            <code>persistent</code>.
            These allow off-heap plot data caching, so that interactive
            plots for very large data sets may be made without running
            of of memory.
            The <code>persistent</code> option additionally means that
            results of expensive data read operations can be cached
            between invocations.</li>
        <li>The <ref id="layer-line"><code>line</code></ref> layer's 
            <code>sortaxisN</code> parameter now accepts the option
            <code>time</code> rather than <code>X</code> (or <code>Y</code>)
            when used with <code>plot2time</code>.</li>
        <li>Improved plot axis labelling in LaTeX mode, e.g. in LaTeX write
            "3x10<sup>6</sup>" not "<code>3e6</code>".</li>
        <li>Add new option <code>anchorN=center</code> to the
            <ref id="layer-label"><code>label</code></ref> plot layer.</li>
        <li>Add parameters <code>minsize</code> and <code>minshape</code>
            to <ref id="layer-poly4"><code>poly4</code></ref>
            plot layer type.</li>
        <li>Improve accuracy when drawing large HEALPix tile boundaries
            in some cases for
            <ref id="layer-skydensity"><code>skydensity</code></ref> and
            <ref id="layer-healpix"><code>healpix</code></ref>
            sky plot layers.</li>
        </ul>
        </p></dd>
    <dt>Other minor enhancements:</dt>
    <dd><p>
        <ul>
        <li>Basic support for SIA version 2 as well as version 1 in
            <ref id="cone"><code>cone</code></ref> and
            <ref id="coneskymatch"><code>coneskymatch</code></ref> commands
            (new option <code>servicetype=sia2</code>).</li>
        <li>Add new conversion functions <code>*ToUnixSec</code> to
            <ref id="Times">Times</ref> class.</li>
        <li>Add new functions <code>midLon</code> and <code>midLat</code>
            to <ref id="Sky">Sky</ref> class.</li>
        <li>Slightly improve sampling accuracy of
            <ref id="pixsample"><code>pixsample</code></ref> output.</li>
        </ul>
        </p></dd>
    <dt>Bug and misfeature fixes:</dt>
    <dd><p>
        <ul>
        <li>Fix bug in <ref id="layer-skygrid"><code>SkyGrid</code></ref>
            plot layer that made it ignore the <code>viewsys</code> parameter,
            always assuming equatorial so plotting the wrong grid lines
            for non-equatorial view sky systems.  Note this bug applied
            only to STILTS, not TOPCAT.</li>
        <li>Fix regression bug since last release that refused to make
            dataless plots.</li>
        <li>Fix <ref id="layer-polygon"><code>polygon</code></ref> plots in
            <ref id="plot2sphere"><code>plot2sphere</code></ref> - this
            combination was failing to plot anything, now it works.</li>
        <li>FITS ASCII table extensions with TFORM values of <code>In</code>
            are now treated as 64-bit integers for <code>n&gt;=10</code>
            rather than <code>n&gt;10</code>.</li>
        <li>Modify <code>taplint</code> processing of
            <code>QUERY_STATUS</code> declarations;
            overflow may now be declared before,
            as well as after, the table.</li>
        <li>Slightly improve plot axis labelling for small numbers;
            remove unnecessary decimal point in scientific notation
            in some cases.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.2-2">Version 3.2-2 (24 August 2020)</dt>
<dd><p>
    <dl>
    <dt>New Functionality:</dt>
    <dd><p>
        <ul>
        <li>SVG (Scalable Vector Graphics) is now one of the supported
            <ref id="graphicExporter">graphic output formats</ref>
            for plot export.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes and minor enhancements:</dt>
    <dd><p>
        <ul>
        <li>Fix Healpix trouble at large angles, e.g. sky crossmatch failure
            with match radius &gt;6 degrees.</li>
        <li>Fix ECSV output bug: encoding was incorrect for metadata
            scalars with certain non-alphanumeric first characters,
            leading to invalid YAML.</li>
        <li>Votlint now warns about use of vocabulary terms marked as
            <em>preliminary</em> or <em>deprecated</em> as well as
            unknown ones (although currently no such terms exist
            in relevant vocabularies).</li>
        <li>Remove some unhelpful per-column metadata items from
            ECSV output.</li>
        <li>Improve seeding of the expression language <code>$random</code>
            special value;
            it should now be less dependent on JVM details.</li>
        <li>TAP <code>curl(1)</code> equivalent logging flag error fixed
            (write <code>--compressed</code> not <code>--compress</code>).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.3">Version 3.3 (23 October 2020)</dt>
<dd><p>
    <dl>
    <dt>Plot server</dt>
    <dd><p>A <ref id="PlotServlet">plot server</ref> has been added
        that can serve interactive plots in web applications
        for server-side tables.
        Documentation for the <ref id="server"><code>server</code></ref>
        command has been mostly moved into new
        <ref id="serverMode">Server Mode</ref> section.
        </p></dd>
    <dt>Bug fixes</dt>
    <dd><p>
        <ul>
        <li>Upgrade cds-healpix-java library to v0.28_1; avoid occasional
            sky crossmatch failures (termination with error).</li>
        <li>Fix minor plotting bug that could cause white points to
            be invisible.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4">Version 3.4 (11 January 2021)</dt>
<dd><p>In this version, the table handling library STIL has
    been upgraded to v4.0, which enables some enhancements
    including multithreading and new I/O handler features.
    <dl>
    <dt>Notable new functionality:</dt>
    <dd><p>
        <ul>
        <li>Auto file format detection now examines filenames to help
            guess format; this means that e.g. for CSV files named
            with the "<code>.csv</code>" extension, it is no longer
            necessary to specify <code>ifmt=csv</code>.</li>
        <li>Several operations can now run in parallel,
            so should execute faster
            for large datasets on multi-core machines.
            Parallelised operations are the output modes
            <ref id="mode-stats">stats</ref> and
            <ref id="mode-count">count</ref>,
            the filters <ref id="stats">stats</ref> and
            <ref id="sort">sort</ref>,
            and the task <ref id="tskymap">tskymap</ref>.
            The <code>stats</code> and <code>sort</code> filters have
            <code>-[no]parallel</code> flags that can be used to select
            between sequential and parallel execution.
            Parallel data caching for plots is also available
            using the <code>storage=parallel</code> option,
            but this is currently experimental and not generally recommended.
            Future releases will parallelise other operations,
            including matching.</li>
        <li>I/O handlers can now in some cases take parenthesised options,
            e.g. <code>ofmt=votable(version=V12,format=BINARY)</code>.
            For examples see individual handler documentation,
            e.g. the <ref id="outVotable">VOTable output handler</ref> and
            <ref id="inEcsv">ECSV input handler</ref>.</li>
        <li><ref id="TableScheme">Scheme specifiers</ref> can now be used
            to specify input tables, to read tables not based on
            external files.  Options currently provided include
            <ref id="scheme-loop">simple sequence</ref>,
            <ref id="scheme-skysim">simulated sky</ref> and
            <ref id="scheme-attractor">strange attractor</ref> data.</li>
        <li>Improved documentation of I/O formats in this document:
            full docs are now provided in <ref id="ioFormats"/>
            for all formats rather than summaries
            with pointers to <docxref doc="sun253"/>.
            The <ref id="io">Table I/O</ref> section has been somewhat
            reorganised and tidied up.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements:</dt>
    <dd><p>
        <ul>
        <li>When the <ref id="stats">stats</ref> filter calculates quantiles,
            the <code>-qapprox</code> flag can be supplied to use an
            (approximate) algorithm that will not run out of memory.</li>
        <li>Add <code>combine</code> parameter to
            <ref id="tcube"><code>tcube</code></ref>,
            so it can produce generalised weighted maps.</li>
        <li>Add new options <code>acos</code> and <code>cos</code>
            for colour scaling options
            <code>auxfunc</code>, <code>densefunc</code> etc;
            these provide linear-like stretch functions with
            steeper/flatter ends, which may be useful for shading
            by quantities with most variation near to/far from the
            middle of the range.</li>
        <li>The <ref id="progress">progress</ref> filter no longer
            slows down pipelines to any noticeable extent.</li>
        <li>COOSYS and TIMESYS attributes are now preserved
            during VOTable I/O for table PARAMs
            (as well as for FIELDs, which was already the case).</li>
        <li>Tidy up open files in the table concatenation command
            <ref id="tcat"><code>tcat</code></ref>;
            with <code>lazy=true</code>
            it should no longer fail with "Too many open files".</li>
        <li>The <ref id="every">every</ref> filter has a new
            <code>-exact|-approx</code> flag;
            <code>-approx</code> may be required for certain operations
            to run in parallel.</li>
        <li>Slightly improved documentation for
            <ref id="cdsskymatch"><code>cdsskymatch</code></ref>.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix bug that could give unhelpful table load error message
            for very short non-FITS files.</li>
        <li>Fix broken example in
            <ref id="tmatchn-examples"><code>tmatchn</code>
                                       documentation</ref>.</li>
        <li>Fix <ref id="taplint"><code>taplint</code></ref> to cope with
            VOSI1.1 <code>detail=min</code>-style response from
            <code>/tables</code> endpoint; avoid spurious
            <code>E-TMC-CM21</code>/<code>E-TMC-FM21</code> errors.</li>
        <li>Cope better with infinite values in aux plot coordinates.</li>
        <li>Fix histogram ranging bug
            (plot failure under certain circumstances).</li>
        <li>Upgrade JEL to v2.1.2: bugfix plus some efficiency improvements
            in object creation.</li>
        <li>Avoid inserting <code>NULL_VALUE</code> custom metadata entries
            into ECSV output.</li>
        <li>Make <ref id="taplint"><code>taplint</code></ref> changes for
            later edits to ObsCore-1.1 Erratum #1
            (<code>dataproduct_type</code> and
             <code>dataproduct_subtype</code> UCDs are now
             <code>meta.code.class</code>).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-1">Version 3.4-1 (10 June 2021)</dt>
<dd><p>
    <dl>
    <dt>File formats:</dt>
    <dd><p>
        <ul>
        <li>Apache Parquet format is now supported for
            <ref id="inParquet">input</ref> and
            <ref id="outParquet">output</ref>
            (note not available in all configurations).</li>
        <li>AAS Machine-Readable Table (MRT) format is now supported
            for <ref id="inMrt">input</ref>.</li>
        <li>ECSV format <ref id="inEcsv">input</ref> and
            <ref id="outEcsv">output</ref> handlers
            are upgraded to version 1.0 of the ECSV format,
            meaning they can now read and write array-valued columns.</li>
        <li>Add configuration option <code>header</code> for
            <ref id="inCsv">CSV input handler</ref>,
            to indicate whether header line is present.</li>
        <li>Add configuration option <code>maxSample</code> for
            <ref id="inCsv">CSV</ref> and <ref id="inAscii">ASCII</ref>
            input handlers to reduce 2-pass read time.</li>
        <li>Variable-length array-valued columns in FITS tables
            (<code>P</code>/<code>Q</code> descriptors)
            can now be read even in compressed or streamed input.</li>
        </ul>
        </p></dd>
    <dt>Other new functionality:</dt>
    <dd><p>
        <ul>
        <li>Add new plot layers to <ref id="plot2plane">plot2plane</ref>
            for plotting data (such as spectra or time series)
            in array-valued columns:
            <ref id="layer-lines">lines</ref>,
            <ref id="layer-marks">marks</ref>,
            <ref id="layer-yerrors">yerrors</ref> and
            <ref id="layer-xyerrors">xyerrors</ref>.</li>
        <li>Add new functions to class
            <ref id="Arrays">Arrays</ref> for working with array values:
            <code>arrayFunc</code>, <code>intArrayFunc</code>,
            <code>sequence</code>, <code>constant</code>.</li>
        <li>Add new table scheme <ref id="scheme-test">test</ref>.</li>
        <li>The <code>cumulative</code> option in
            <ref id="layer-histogram">histogram</ref>-like plots
            can now take the values
            <code>none</code>/<code>forward</code>/<code>reverse</code>,
            not just <code>true</code>/<code>false</code>.</li>
        <li>Introduce the <code>aitoff0</code> option alongside
            <code>aitoff</code> as a value for the
            <ref id="plot2sky">plot2sky</ref>
            <code>projection</code> parameter.
            In the same way as <code>car</code>/<code>car0</code>,
            this gives an Aitoff projection but with the longitude=0
            line at the left/right edge instead of the center.</li>
        <li>Modify <ref id="tcube">tcube</ref> so that it accepts
            expression language expressions rather than just column name/ID
            values for the <code>cols</code> and <code>scale</code>
            parameters.</li>
        </ul>
        </p></dd>
    <dt>Taplint changes:</dt>
    <dd><p>
        <ul>
        <li>Add new <ref id="taplint">taplint</ref>
            stage <code>LOC</code>, for ObsLocTAP validation.
            This release validates against the draft
            standard version PR-ObsLocTAP-20210609.</li>
        <li>Taplint stage <code>TMV</code> now validates
            against (a pre-REC version of) version 1.2 of the
            VODataService schema instead of version 1.1.</li>
        <li>Downgrade taplint report for accepting unknown LANG from
            <code>E_DSUC</code> to <code>W_OLNG</code>.</li>
        <li>Relax taplint timestamp upload round-tripping requirements;
            folding <code>adql:TIMESTAMP</code>-&gt;<code>timestamp</code>
            no longer provokes error <code>E-UPL-TMCX</code>.</li>
        <li>Relax some taplint type checking for TAP_SCHEMA columns
            of TAP 1.0 services (see TAP 1.0 Erratum #3);
            report <code>E-TMS-TSCT</code> is downgraded to
            <code>W-TMS-TSCT</code>.</li>
        <li>Taplint <code>W_EXVC</code> TOPCAT compatibility warnings
            are downgraded to <code>I_EXVT</code>.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and workarounds:</dt>
    <dd><p>
        <ul>
        <li>The <ref id="outFits">fits-var</ref> output handler now avoids
            use of the THEAP keyword (no pre-heap gap is written).
            Heap padding is legal FITS, but bugs in other 
            FITS software mean that some third party components
            (including <code>fverify</code> in FTOOLS e.g. v3.14-3.50)
            have problems with such files.</li>
        <li>Fix <ref id="server">server</ref>
            to work with scheme-specified tables having a
            non-default <code>tablefactory</code> parameter.</li>
        <li>JDBC output no longer attempts to create VARCHAR(0) columns.</li>
        <li>Output of <ref id="cache"><code>cache</code></ref> filter
            now retains no reference to input table, so it can be used
            to manage deeply nested tables in (Jy)STILTS pipelines.</li>
        <li>JyStilts now has access to the same table schemes as STILTS.</li>
        <li>Collapse whitespace in some table metadata description items
            in output form <code>meta</code> filter and output mode.
            This prevents unwanted linebreaks as well as tidying
            whitespace-formatted values.</li>
        <li>Space-delimited ECSV files now write empty fields quoted.</li>
        <li>Unknown or unsupported column datatype values in ECSV files
            are now treated like <code>string</code> rather than
            causing table read failure.</li>
        <li>Empty strings in FITS 1-character columns are now returned as
            blank values rather than ASCII NUL ('<code>\0</code>').</li>
        <li>Undersized, including zero-length, strings written to FITS
            columns are now by default terminated with an ASCII NUL
            rather than in some cases padded with spaces.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix regression bug in previous release (v3.4) that broke use of
            <code>jdbc:</code> URLs.</li>
        <li>Fix regression bug in previous release (v3.4) affecting
            <code>text</code>, <code>ascii</code> and <code>ipac</code>
            output formats, that could cause cached table output
            (e.g. from filters <code>cache</code> or <code>random</code>)
            to contain many repeated copies of the same row.</li>
        <li>Upgrade cds-healpix-java library to v0.29.3;
            this avoids occasional sky crossmatch failures
            (termination with error) introduced at STILTS v3.2-1.</li>
        <li>Fix bug in <ref id="addcol"><code>addcol</code></ref> filter
            that could evaluate the row index
            (<ref id="jel-special"><code>$index/$0</code></ref>)
            incorrectly during parallel processing.</li>
        <li>Fix <code>plot2sky</code> bug that got
            <code>clon</code>/<code>clat</code> positioning wrong
            for some projections such as <code>CAR0</code>.</li>
        <li>Fix issue with cumulative
            <ref id="layer-histogram">histograms</ref>;
            bars beyond the last sample are now displayed with total value
            not zero.</li>
        <li>Fix bugs that meant writing long (&gt;2Gb)
            <ref id="outFits">fits-var</ref> files could output
            incorrect/corrupted FITS.</li>
        <li>Fix long-standing file caching bug; mostly seemed to
            affect large (&gt;2Gb) streams.</li>
        <li>Fix documentation bug in <ref id="cone">cone</ref>;
            output parameters <code>out</code>, <code>ofmt</code>,
            <code>ocmd</code> and <code>omode</code> are now listed.</li>
        <li>Fix optional argument processing bug in JyStilts
            (optional named arguments like <code>start</code> in
            <code>tloop</code> were being ignored).</li>
        <li>Fix <ref id="tcube">tcube</ref> bug that caused problems with
            some input filter (<code>icmd</code>) processing.</li>
        <li>Fix long-standing logic error in ASCII/CSV input handler that
            could misidentify column types and cause read failures.</li>
        <li>Fix failure when trying to plot with
            <code>auxfunc=histogram</code> and
            explicitly set <code>auxmin</code>/<code>auxmax</code> bounds
            outside of data range.</li>
        <li>FITS TZERO headers are now written correctly with numeric values
            rather than string values.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-2">Version 3.4-2 (15 October 2021)</dt>
<dd><p>
    <dl>
    <dt>Taplint and Votlint changes:</dt>
    <dd><p>
        <ul>
        <li>New stage <code>UUC</code> added to
            <ref id="taplint">taplint</ref>.
            This checks units
            (against the <webref url="https://www.ivoa.net/documents/VOUnits/"
                                 >VOUnits</webref> standard)
            and UCDs
            (against the <webref url="https://www.ivoa.net/documents/UCD1+/"
                                 >UCD1+</webref> standard
             and legacy UCD1 list).</li>
        <li>New stage <code>EPN</code> added to taplint.
            This checks EPN-TAP (<code>*.epn_core</code>) tables
            against some version of
            <webref url="https://www.ivoa.net/documents/EPNTAP/"
                    >EPN-TAP</webref> 2.0
            (currently PR-EPNTAP-2.0-20211012).
            Since EPN-TAP is still in draft at time of writing,
            this stage is not currently run by default.</li>
        <li>Modify taplint <code>stages</code> parameter usage;
            you can now add and remove stages individually using
            <code>+XXX</code>/<code>-XXX</code> syntax.</li>
        <li>Taplint now does a better job of categorising VOTable parse
            errors, so multiple repeats of similar messages can be avoided.
            Ad-hoc VOTable error codes have changed from
            "<code>VOxx</code>" to "<code>Yxxx</code>".</li>
        <li><ref id="votlint">Votlint</ref>,
            hence also taplint, now checks values in
            FIELD/PARAMs marked with <code>xtype</code> attributes for
            conformance to rules from
            <webref url="https://www.ivoa.net/documents/DALI/20170517/"
                    >DALI 1.1</webref> sec 3.3.</li>
        <li>Taplint <code>MDQ</code> stage now reports mismatches
            between declared and result Units, UCDs, Xtypes, and Utypes
            as well as datatypes.</li>
        <li>Taplint and <ref id="datalinklint">datalinklint</ref>
            now identify themselves as validators
            using the <code>User-Agent</code> token "<code>(IVOA-test)</code>"
            rather than "<code>(IVOA-validate)</code>" as before;
            this follows the published content of the
            <webref url="https://www.ivoa.net/documents/Notes/softid/"
                    >SoftID-1.0</webref> IVOA Note.</li>
        <li>Taplint now examines the service <code>Server</code> header
            and tries to check it for VO component identification
            as recommended by <code>SoftID</code>.</li>
        <li>Votlint now has a <code>maxrepeat</code>
            parameter like <code>taplint</code>
            (previously the repeat count was hard-wired to 4).</li>
        <li>Votlint now does a better job of identifying similar errors,
            so output suppressing repeated messages can be more compact.
            Suppression of repeated messages is reported.</li>
        <li>Votlint now optionally checks the content of <code>unit</code>
            and <code>ucd</code> attributes, checking against the
            VOUnits and UCD1+/UCD1 standards respectively.</li>
        <li>Distinguish <code>votlint</code> reports for recognised and
            unrecognised (e.g. VODML) elements in foreign XML namespaces.</li>
        <li>Add timestamp line at start of <code>taplint</code> output.</li>
        </ul>
        </p></dd>
    <dt>Other new functionality:</dt>
    <dd><p>
        <ul>
        <li>A new parameter <code>thick</code> has been added to several
            plot layer types to enable lines thicker than a single pixel
            when drawing error bars, arrows, outlines, pair links etc:
            <ref id="layer-xyvector"><code>xyvector</code></ref>,
            <ref id="layer-xyerror"><code>xyerror</code></ref>,
            <ref id="layer-sizexy"><code>sizexy</code></ref>,
            <ref id="layer-xyellipse"><code>xyellipse</code></ref>,
            <ref id="layer-xycorr"><code>xycorr</code></ref>,
            <ref id="layer-link2"><code>link2</code></ref>,
            <ref id="layer-poly4"><code>poly4</code></ref>,
            <ref id="layer-polygon"><code>polygon</code></ref>,
            <ref id="layer-area"><code>area</code></ref>,
            <ref id="layer-yerrors"><code>yerrors</code></ref>,
            <ref id="layer-xyerrors"><code>xyerrors</code></ref>,
            <ref id="layer-skyvector"><code>skyvector</code></ref>,
            <ref id="layer-skyellipse"><code>skyellipse</code></ref>,
            <ref id="layer-skycorr"><code>skycorr</code></ref>,
            <ref id="layer-xyzvector"><code>xyzvector</code></ref>,
            <ref id="layer-xyzerror"><code>xyzerror</code></ref>,
            <ref id="layer-yerror"><code>yerror</code></ref>.</li>
            <li>New functions in class 
                <ref id="VO">VO</ref>
                to check syntax of UCD and VOUnit strings.</li>
        <li>New <code>areatype</code> value <code>UNIQ</code> for plotting
            single HEALPix tiles e.g. from MOC files in
            <ref id="layer-area">area</ref> (and similar) layer type.</li>
        <li>New <ref id="mode-checksum"><code>checksum</code></ref>
            table output mode.</li>
        <li>Add Julian Day manipulation functions <code>jdToMjd</code>
            and <code>mjdToJd</code> to class
            <ref id="Times">Times</ref>.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>The <code>fast</code> parameter has been withdrawn from
            polygon-plotting plot types
            <ref id="layer-area"><code>area</code></ref>,
            <ref id="layer-polygon"><code>polygon</code></ref>,
            <ref id="layer-poly4"><code>poly4</code></ref>;
            the shape-filling algorithm has been improved, and it's no
            longer necessary to choose between speed and accuracy.</li>
        <li>Replace the <code>julianToUnixSec</code> function in class
            <ref id="Times">Times</ref>, which didn't do what it said it did,
            with <code>jdToUnixSec</code>.</li>
        <li>The <code>mjdToIso</code> and <code>mjdToDate</code> functions
            in class <ref id="Times">Times</ref>
            now prepend the string
            "<code>(BCE)</code>" to dates before the Common Era.</li>
        <li>Guess meaning for some non-standard COORDSYS values in
            HEALPix-FITS files, e.g. allow "GALACTIC" instead of "G".</li>
        <li>Revert to sequential processing in some cases for
            HEALPix plot layer to reduce resource usage.</li>
        <li>Columns that are all blank in ASCII-like tables
            (CSV, ASCII, TST) are now interpreted as type String
            not boolean.</li>
        <li>Files compressed using multi-stream bzip2 compression
            (e.g. <code>pbzip2</code> output) are now supported
            alongside single-stream bzip2.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix serious threading bug that could return nonsense values
            from fixed-length string fields during parallel processing,
            for instance statistics calculation,
            of large cached or randomised tables.</li>
        <li>Upgrade of cds-healpix-java library to v0.30.2; this fixes
            some HEALPix cell plotting bugs.</li>
        <li>Fix aux ranging bug that meant <code>auxmax</code> without
            <code>auxmin</code> was not honoured in plot commands.
            This was a regression bug introduced at v3.1-5.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-3">Version 3.4-3 (31 January 2022)</dt>
<dd><p>
    <dl>
    <dt>New functionality:</dt>
    <dd><p>
        <ul>
        <li>The <ref id="inPds4">PDS4</ref>
            (NASA's Planetary Data System v4) file format is now
            supported for input tables.</li>
        <li>New task <ref id="tgridmap">tgridmap</ref> writes flexible
            N-dimensional density maps/histograms as tables.</li>
        <li>The <code>cols</code> parameter in
            <ref id="tskymap">tskymap</ref> is generalised to allow
            more flexible calculation of gridded quantities,
            specifically calculations using different aggregation methods
            for different quantities at the same time.</li>
        <li>New plot2plane layers <ref id="layer-statline">statline</ref>
            and <ref id="layer-statmark">statmark</ref>, to plot
            quantities like the mean of array-valued input columns.</li>
        <li>New class
            <ref id="Randoms">Randoms</ref>
            contains pseudo-random number generation functions.
            Special token <code>$random</code> is now deprecated.</li>
        <li>New quantities
            <code>ArrayNGood</code>, <code>ArraySum</code>,
            <code>ArrayMean</code>, <code>ArrayStDev</code>
            added to <ref id="stats"><code>stats</code></ref> filter,
            allowing per-element statistics calculations for
            fixed-length numeric array data.</li>
        <li>New <ref id="taplint">taplint</ref> parameter <code>tables</code>
            to restrict which tables are tested.</li>
        <li>Provide quantile options for the <code>combiner</code>
            parameters of some commands.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>Add new colour map
            <webref url="https://xkcd.com/2537/">Painbow</webref>.</li>
        <li>Improve identification of TIME_TT2000 columns as time values 
            in certain CDF files.</li>
        <li>Improved autoranging for
            <ref id="layer-gaussian">gaussian</ref> plots.</li>
        <li>Minor improvements to aggregated column output metadata.</li>
        <li>JyStilts is now intended to be run using Jython 2.7.2,
            which means it works under Java 17, unlike Jython 2.5.3
            which was the previous target version.
            The JyStilts classes haven't changed, but the distributed
            <code>jystilts.jar</code> file now comes with Jython 2.7.2.</li>
        <li>In the starjava distribution, the <code>jystilts</code> script
            now uses an internal copy of Jython 2.7.2, it no longer requires
            an external Jython installation.</li>
        <li>Taplint UUC stage now reports UCD word list versions.</li>
        <li>Adjustments to taplint EPN-TAP validation:
            track draft standard changes, fix bugs,
            add UCD check for measurement_type column content.
            This version currently corresponds to
            PR-EPNTAP-2.0-20211022.</li>
        <li>Make a couple of adjustments to
            <ref id="graphicExporter">SVG graphics output</ref>:
            output is now to a bare <code>svg</code> element
            (no XML or DOCTYPE declaration),
            and a <code>viewBox</code> attribute is included which
            may improve scaling behaviour in some contexts.</li>
        <li>The monolithic stilts.jar file and stilts_jars.zip archive
            now contain the <code>stilts.version</code> file at the
            top level, rather than just buried in
            <code>uk/ac/starlink/ttools/</code>.</li>
        <li>The <code>seqcol</code> column in output from the
            <ref id="tcat"><code>tcat</code></ref> and
            <ref id="tcatn"><code>tcatn</code></ref> commands
            is now of type (32-bit) integer not (16-bit) short.</li>
        <li>Permit leading '<code>s</code>' in ASCII-MOC area specifications,
            following MOC 2.0.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes:</dt>
    <dd><p>
        <ul>
        <li>Fix bugs in parsing <code>MOC-ASCII</code> strings in
            <ref id="layer-area">area</ref> plot;
            trailing depth specifier was interpreted as cell index
            and some cells near end of MOC were omitted/misshapen.</li>
        <li>Fix failure when attempting to read unsigned 32-bit integer
            values from parquet files.</li>
        <li>Bugfix update of JCDF to v1.2-4.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-4">Version 3.4-4 (6 April 2022)</dt>
<dd><p>
    <dl>
    <dt>Performance</dt>
    <dd><p>
        <ul>
        <li>Substantial I/O performance improvements,
            mainly for FITS and VOTable formats e.g.:
            writing to FITS 2x,
            reading FITS from a stream 2x,
            reading VOTable with inline BINARY/BINARY2 4x,
            writing VOTable with inline BINARY/BINARY2 2x,
            writing VOTable with TABLEDATA 1.5x.</li>
        <li>New parameter <code>runner</code> in matching commands
            <ref id="tmatch2">tmatch2</ref> etc which can be set to
            <code>parallel</code> for multi-threaded execution.
            This should speed up large matches on multi-core machines.
            This is believed to work correctly, but since it is less
            well tested than the legacy sequential mode, at this release
            sequential processing is still the default.</li>
        <li>Other matching performance improvements.</li>
        <li>FITS I/O is now all done internally, there is no longer
            a dependency on the nom.tam.fits package.</li>
        </ul>
        </p>
        </dd>
    <dt>New functionality</dt>
    <dd><p>
        <ul>
        <li>New command <ref id="xsdvalidate"><code>xsdvalidate</code></ref>
            added for validating against XML schemas.</li>
        <li>New class <ref id="Bits">Bits</ref>
            with bit manipulation functions
            <code>bitCount</code>, <code>hasBit</code>,
            <code>toBinary</code>, <code>fromBinary</code>.</li>
        <li>Add configuration options
            <code>compact</code> and <code>encoding</code> to
            <ref id="outVotable">VOTable output handler</ref>.
            By default thin (&lt;=4 column) TABLEDATA VOTables
            are now written in "compact" mode, using reduced whitespace.</li>
        <li>FITS BINTABLE headers used as table parameters
            now support FITS 4.0 long-string syntax (CONTINUE records).</li>
        <li>FITS header values of the form "(a,b,c,...)" are now interpreted
            where possible as numeric arrays; this works for long-string
            values (CONTINUE records) as well.</li>
        </ul>
        </p></dd>
    <dt>Bug Fixes and workarounds</dt>
    <dd><p>
        <ul>
        <li>Upgrade JEL to v2.1.3-pre1.
            This fixes a bug that caused evaluation failure when comparing
            a String against null.</li>
        <li>Taplint now reports absent TAP service with an error code
            rather than failing with an exception.</li>
        <li>Upgrade Unity library to 1.1 pre-release
            and improve VOUnits validation reporting
            (new status "GUESSED_UNIT").</li>
        <li>Fix taplint bug: EPN validation failed for
            <code>spatial_frame_type=none</code>.</li>
        <li>ECSV format now preserves table name.</li>
        <li>FITS BINTABLE reader now copes with (illegal?) embedded spaces
            in TDIMn headers.</li>
        <li>Adjust MRT null handling; "-" in a single-character field no
            longer interpreted as null.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-5">Version 3.4-5 (10 June 2022)</dt>
<dd><p>
    <dl>
    <dt>New Functionality</dt>
    <dd><p>
        <ul>
        <li>New plot2plane layer
            <ref id="layer-arrayquantile">arrayquantile</ref>,
            that can plot e.g. medians of multiple per-row spectra
            with arbitrary wavelength coordinates.</li>
        <li>X/Y array-valued plot layer types
            <ref id="layer-lines">lines</ref>,
            <ref id="layer-marks">marks</ref>,
            <ref id="layer-yerrors">yerrors</ref>,
            <ref id="layer-xyerrors">xyerrors</ref>,
            <ref id="layer-statline">statline</ref> and
            <ref id="layer-statmark">statmark</ref>
            will now accept a missing <code>xs</code> or <code>ys</code>
            array coordinate, and assume a suitable linear sequence.</li>
        <li>The <ref id="colid-list"><code>&lt;colid-list&gt;</code></ref>
            syntax used by
            <ref id="keepcols"><code>keepcols</code></ref>,
            <ref id="delcols"><code>delcols</code></ref> etc
            now allows column ranges to be specified with the syntax
            <code>&lt;first-colid&gt;-&lt;last-colid&gt;</code>.</li>
        <li>The <ref id="assert"><code>assert</code></ref> filter now takes
            optional <code>&lt;msg-expr&gt;</code> argument.</li>
        <li>Add offset options <code>xoff</code> and <code>yoff</code>
            for <ref id="layer-label"><code>label</code></ref>
            plot layer type.</li>
        <li>Minor <ref id="taplint"><code>taplint</code></ref> EPN stage
            updates corresponding to latest EPN-TAP draft;
            now matches PR-EPNTAP-20220420.</li>
        </ul>
        </p></dd>
    <dt>Performance</dt>
    <dd><p>
        <ul>
        <li>Sky crossmatching performance improvements.</li>
        <li>Reduce number of file mapping calls by FITS readers.</li>
        </ul>
        </p></dd>
    <dt>Bug Fixes</dt>
    <dd><p>
        <ul>
        <li>Fix FITS parsing issue that could result in StackOverflowError
            for long array-valued headers.</li>
        <li>Fix bugs in <ref id="layer-statline">statline</ref
                      >/<ref id="layer-statmark">statmark</ref> layers
            that incorrectly treated blank array values
            and negative array values on logarithmic axes.</li>
        <li>Fix bug in multi-threaded read of string columns from
            colfits files.</li>
        <li>Fix NullPointerException failure in taplint UUC stage or
            votlint for unknown units recognised only in non-VOUnit
            syntaxes (e.g. "Crab").</li>
        <li>Fix legend positioning bug for <code>label</code>
            layer type.</li>
        <li>Aitoff sky projections are now more correctly documented
            as Hammer-Aitoff (though not renamed in the UI).</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-6">Version 3.4-6 (8 July 2022)</dt>
<dd><p>
    <dl>
    <dt>Enhancements</dt>
    <dd><p>
        <ul>
        <li>New command <ref id="arrayjoin"><code>arrayjoin</code></ref>
            added.</li>
        <li>Modify functions <code>add</code>, <code>subtract</code>,
            <code>multiply</code>, <code>divide</code> in class
            <ref id="Arrays">Arrays</ref>;
            these now all take either two array arguments or an array
            and a scalar in either order.</li>
        <li>New function <code>dotProduct</code> in class
            <ref id="Arrays">Arrays</ref>.</li>
        <li>Provide <code>value*</code> functions for
            <ref id="jel-colref">column references</ref>
            to strangely-named columns.</li>
        <li>Minor <ref id="taplint"><code>taplint</code></ref> EPN stage
            update; now corresponds to REC-EPNTAP-2.0.</li>
        </ul>
        </p></dd>
    <dt>Bug Fix</dt>
    <dd><p>
        <ul>
        <li>Fix FITS output so it doesn't fail when attempting to write
            metadata with non-ASCII Unicode characters.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-7">Version 3.4-7 (5 October 2022)</dt>
<dd><p>
    <dl>
    <dt>Performance:</dt>
    <dd><p>
        <ul>
        <li>Crossmatching performance improvements:
            sky matching does better pre-selection of potential matches
            based on sky region,
            post-processing row sorting etc accelerated,
            various other steps parallelised.
            Large matches might typically be about twice as fast
            as before.</li>
        <li>By default, matching commands like
            <ref id="tmatch2"><code>tmatch2</code></ref>
            now run in parallel.</li>
        <li>The usage of the
            <ref id="tmatch2-runner"><code>runner</code></ref> parameter,
            used to control parallel execution of matching commands like
            <code>tmatch2</code>, has changed slightly:
            options <code>parallel&lt;n&gt;</code> and
            <code>parallel-all</code> are added,
            while option <code>parallel</code>, the new default,
            is now limited to a fixed number of processors
            (currently 6).</li>
        <li>Add new parameter
            <ref id="tskymap-runner"><code>runner</code></ref> to
            <ref id="tskymap">tskymap</ref> and
            <ref id="tgridmap">tgridmap</ref>;
            this allows to control parallel execution,
            since sequential may be faster than the default parallel option
            for large files on spinning disks.</li>
        <li>Add new option "<code>time</code>" to the
            <ref id="tmatch2-progress"><code>progress</code></ref> parameter
            of matching commands like <code>tmatch2</code>.</li>
        </ul>
        </p></dd>
    <dt>New functionality:</dt>
    <dd><p>
        <ul>
        <li>Add new plot layer type
            <ref id="layer-handles"><code>handles</code></ref>
            to mark a reference position for X/Y array data.</li>
        <li>New function <code>sequence(n,start,step)</code> in
            class <ref id="Arrays-sequence">Arrays</ref>.</li>
        <li>New functions <code>tfcatStatus</code> and
            <code>tfcatMessage</code> in class
            <ref id="VO-tfcatStatus">VO</ref> for validating instances of
            the Time-Frequency Radio Catalogue format.</li>
        <li>Provide new option <code>TFCAT</code> for
            <code>areatype</code> parameter
            in <ref id="layer-area">area</ref>-like plots,
            giving partial support for TFCat shape descriptions.</li>
        <li>Filter <ref id="fixcolnames"><code>fixcolnames</code></ref>
            now deduplicates column names
            as well as fixing up their syntax.</li>
        <li>Fix relevant filters to issue a WARNING if new column names
            clash with existing ones.</li>
        <li>Add new section listing <ref id="Shaders">Colour Maps</ref>
            to this document.</li>
        <li>Add some new colour maps from 
            <webref url="https://github.com/1313e/CMasher">CMasher</webref>:
            <code>cosmic</code>, <code>ember</code>, <code>gothic</code>,
            <code>rainforest</code>, <code>voltage</code>,
            <code>bubblegum</code>, <code>gem</code>, <code>chroma</code>,
            <code>neon</code>, <code>tropical</code> (sequential);
            <code>guppy</code>, <code>iceburn</code>,
            <code>redshift</code>, <code>pride</code> (diverging);
            <code>infinity</code> (cyclic).</li>
        </ul>
        </p></dd>
    <dt>Bug fixes and workarounds:</dt>
    <dd><p>
        <ul>
        <li>Fix VOTable reader and <code>votlint</code> so that
            BINARY/2 VOTables with no columns don't read forever.</li>
        <li>Fix PDS4 reader to accept columns of type
            <code>ASCII_Numeric_Base16</code>
            without the read operation failing.</li>
        <li>Fix bug in <ref id="layer-skygrid">skygrid</ref> layer that
            failed to inherit configuration items from base grid.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-8">Version 3.4-8 (20 April 2023)</dt>
<dd><p>
    <dl>
    <dt>Axis drawing improvements and changes:</dt>
    <dd><p>
        <ul>
        <li>Secondary X and Y axes can now be plotted in
            <ref id="plot2plane"><code>plot2plane</code></ref>;
            see new parameters
            <ref id="plot2plane-x2func"><code>x2func</code>,
                                        <code>y2func</code>,
                                        <code>x2label</code>,
                                        <code>y2label</code></ref>,
            and <ref id="layer-arrayquantile">example plot</ref>.</li>
        <li>Secondary T and Y axes can now be plotted in
            <ref id="plot2time"><code>plot2time</code></ref>;
            see new parameters
            <ref id="plot2time-t2func"><code>t2func</code>,
                                       <code>y2func</code>,
                                       <code>t2label</code>,
                                       <code>y2label</code></ref>,
            and <ref id="layer-spectrogram">example plot</ref>.</li>
        <li>Matching Ticks are now plotted by default on all four sides of
            plots drawn
            by <code>plot2plane</code> and <code>plot2time</code>,
            rather than just on primary (left/bottom) axes.
            This can be configured with the new
            <ref id="plot2plane-shadow"><code>shadow</code></ref>
            parameter.</li>
        <li>Reduce frequency of minor ticks.</li>
        <li>Major tick marks on axes now extend only inside the plot bounds
            not outside.</li>
        <li>Grid lines plotted by
            <code>plot2plane</code>, <code>plot2time</code> and
            <code>plot2sky</code>
            are now all partially transparent and plotted over
            the plot content
            (previously they were opaque and the sky grid was drawn over,
            while the plane and time lines were drawn under,
            the plot content).
            Grid line transparency is controlled by the new
            <ref id="plot2plane-gridtrans"><code>gridtrans</code></ref>
            parameter.</li>
        <li>Add new options <code>ExternalSys</code>/<code>InternalSys</code>
            for <code>plot2sky</code>
            <ref id="plot2sky-labelpos">labelpos</ref> parameter,
            to display lon/lat axis names alongside axis values.</li>
        </ul>
        </p></dd>
    <dt>Other new functionality:</dt>
    <dd><p>
        <ul>
        <li>New task <ref id="tgroup"><code>tgroup</code></ref>
            and filter <ref id="group"><code>group</code></ref>
            to calculate aggregate functions on groups of rows
            (like ADQL/SQL <code>GROUP BY</code>).</li>
        <li>New function
            <ref id="Sky-inSkyEllipse"><code>inSkyEllipse</code></ref>.</li>
        <li>New filter <ref id="constcol"><code>constcol</code></ref>
            for identifying and removing constant-valued table columns.</li>
        <li>Add <ref id="arrayjoin-keepall"><code>keepall</code></ref>
            parameter to the <code>arrayjoin</code> command,
            to configure whether rows with no arrays appear
            in the output.</li>
        <li>Add <ref id="arrayjoin-aparams"><code>aparams</code></ref>
            parameter to <code>arrayjoin</code> command,
            to turn table parameters of loaded tables into
            scalar-valued columns.</li>
        <li>Add <ref id="layer-lines-sortaxisN"><code>sortaxis</code></ref>
            parameter to <code>lines</code> layer,
            to cope with unsorted array data.</li>
        <li>Add configuration options <code>readMeta</code> and
            <code>hierarchicalNames</code> to the
            <ref id="inGbin">GBIN</ref> input handler.</li>
        <li>VOTable service descriptor I/O now preserves
            <code>contentType</code> and <code>exampleURL</code> PARAMs,
            introduced in DataLink 1.1</li>
        <li>Improve <ref id="datalinklint"><code>datalinklint</code></ref>
            validation of semantics terms, now checks online vocabularies
            not just hard-coded ones.</li>
        <li>The <code>datalinklint</code> validator is upgraded
            to PR-DataLink-1.1-20230413.  It now has a
            <ref id="datalinklint-version"><code>version</code></ref>
            parameter.</li>
        <li>Update <code>taplint</code>
            in accordance with
            <webref
            url="https://wiki.ivoa.net/twiki/bin/view/IVOA/DALI-1_1-Erratum-1"
                >DALI 1.1 Erratum #1</webref>;
            example continuation elements without an empty
            resource attribute no longer provoke an Error.</li>
        </ul>
        </p></dd>
    <dt>Workarounds and minor behaviour improvements:</dt>
    <dd><p>
        <ul>
        <li>Modify column width determination in text-like output formats
            (<ref id="outText"><code>text</code></ref>,
             <ref id="outAscii"><code>ascii</code></ref>,
             <ref id="outIpac"><code>ipac</code></ref>)
            to avoid occasional unwanted truncation of formatted values.
            Tables are now read in two passes, the first to establish
            column widths and the second to write the data.
            By default all rows are sampled, but the
            <code>sampledRows</code> option can be configured so that
            only some rows are sampled, which is more like
            the old behaviour.</li>
        <li>Update Ucidy library to v1.3;
            <ref id="VO-ucdStatus">UCD validation</ref> is now done based on
            <webref url="https://www.ivoa.net/documents/UCD1+/20230125/"
                    >v1.5 of the UCD1+ list of terms</webref>.</li>
        <li>If only one single point is plotted on the (default)
            sin projection in
            <ref id="plot2sky"><code>plot2sky</code></ref> without explicit
            <code>clon</code>/<code>clat</code>/<code>radius</code>
            specification, the default view is now zoomed out to the
            whole sky rather than zoomed in to a few milliarcsec.</li>
        <li>Improve <ref id="inEcsv">ECSV reader</ref> performance,
            especially for Gaia DR3 bulk download files
            (which use semi-standard "<code>null</code>" token).</li>
        <li>Write empty string not semi-standard "<code>nan</code>"
            token for NaN in <ref id="outEcsv">ECSV writer</ref>.</li>
        <li>The <ref id="scheme-jdbc">JDBC input scheme</ref>
            should now read columns that are array-valued in the
            database as array values that can be used in STILTS;
            previously they were read as opaque <code>Array</code>
            objects.</li>
        <li>Make FITS and VOTable output handlers robust against
            input tables that declare incorrect row counts.</li>
        <li>The <ref id="inPds4">PDS4 reader</ref> now reads
            <code>Ascii_Numeric_Base16/8/2</code> fields as numeric not string
            (updated pds4-jparser library code).</li>
        <li>Slight change to RESOURCE structure of Primary HDU metadata 
            in multi-table FITS-plus output.
            This fixes a problem in which saved Service Descriptors
            could end up associated with the wrong tables.</li>
        <li>HTTP redirects with response code 308 (Permanent Redirect) are
            now handled in the same way as 307 (Temporary Redirect).</li>
        <li>Withdraw <code>antialias</code> parameter from
            <ref id="layer-lines"><code>lines</code></ref> layer,
            since it didn't really work.</li>
        <li>Improve handling of illegal sky coordinates
            (latitude out of range) in sky matching;
            in v3.4-7 only, bad positions caused match failure,
            now they are just ignored.</li>
        <li>Improve transparency rendering in
            <ref id="layer-skygrid"><code>skygrid</code></ref> layer.</li>
        </ul>
        </p></dd>
    <dt>Bug fixes:</dt>
    <dd><p>
        <ul>
        <li>Fix a nasty bug that could plot
            <ref id="layer-skyvector"><code>skyvector</code></ref>,
            <ref id="layer-skyellipse"><code>skyellipse</code></ref> and
            <ref id="layer-skycorr"><code>skycorr</code></ref>
            shapes incorrectly.
            The numeric value of the shape size coordinates
            (ellipse radii, vector extents) was interpreted in degrees
            before being rescaled as appropriate, so that values of
            a few tens or larger resulted in signficant distortions.
            Now fixed.</li>
        <li>Fix stability issue when using <code>find=best</code>
            (which should be used with caution) in
            <ref id="tmatch2"><code>tmatch2</code></ref> and
            <ref id="tskymatch2"><code>tskymatch2</code></ref>.
            Results may be different in crowded regions, and may also
            have differed between v3.4-6 and v3.4-7,
            but are not obviously more or less correct.</li>
        <li>Fix Cartesian matcher overflow issue;
            for very large ratio between coordinate extent and match radius
            this could cause pathologically slow, though not incorrect,
            crossmatching.</li>
        <li>Fixed bug which failed to plot some markers of multi-marker
            shapes like <ref id="layer-mark2">mark2</ref> when
            multithreaded (i.e. for large tables).
            Regression bug since introduction of multithreaded plotting
            in STILTS v3.2.</li>
        <li>Fix line drawing bug that meant dotted and dashed lines
            were often not displayed correctly.
            This was a regression bug introduced in STILTS 3.1-6.</li>
        <li>Fix auto-range and export bugs that ignored final bar for
            forward cumulative <ref id="layer-histogram">histograms</ref>.</li>
        <li>Fix occasional NullPointerException in taplint EPN stage.</li>
        <li>Avoid some plot failures related to oversized legends and
            undersized plot windows.</li>
        <li>Adjust colour ramp (aux axis) painting to avoid faint white
            stripes seen when vector graphics (e.g. PDF) output
            was rendered in some external viewers.</li>
        <li>Fail with an error rather than silently reading a broken table
            when encountering GaiaTools/zStd-jni bug during
            <ref id="inGbin">GBIN</ref> input.</li>
        <li>Empty/invalid fields encountered by the 
            <ref id="inPds4">PDS4 reader</ref> 
            no longer cause the table read to fail.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-9">Version 3.4-9 (1 November 2023)</dt>
<dd><p>
    <dl>
    <dt>New functionality:</dt>
    <dd><p>
        <ul>
        <li>New plot command
            <ref id="plot2corner"><code>plot2corner</code></ref>.</li>
        <li>The <ref id="AuthManager">authentication handling</ref>
            has changed, and authenticated access can now be made to
            services that comply with the (draft) "SSO_next" proposal
            for advertising authentication in the VO.
            The <code>star.basicauth.user</code
              >/<code>star.basicauth.password</code> system properties
            can no longer be used to set authentication information
            globally for the application, but the
            <code>auth.username</code>/<code>auth.password</code>
            <ref id="sysProperties">system properties</ref>
            can be used in a similar way
            (though this should be done with caution).</li>
        <li>Introduced alternative syntax ("<code>*</code>" separator)
            for <ref id="CombinedMatchEngine">matcher combinations</ref>
            to restrict match separations to the scaled unit sphere.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>Taplint EXA stage now checks RDFa property attributes against the
            <code>http://www.ivoa.net/rdf/examples</code> vocabulary.</li>
        <li>Votlint is aware of VOTable 1.5 and can check against
            some features from WD-VOTable-1.5-20230913
            (currently just vocabularised <code>system</code> and
            <code>refposition</code> attributes of COOSYS).</li>
        <li>Downgrade taplint warning about non-standard
            language features from Warning to Info
            (<code>W-CAP-CULF</code> -&gt; <code>I-CAP-CULF</code>).</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix some IVOID case-sensitivity issues that could result in
            spurious taplint error reports.</li>
        <li>Fix bug which could ignore some points in multi-threaded
            multi-dataset plots.  This bug was introduced in v3.2,
            and was supposed to be fixed at v3.4-8 but wasn't.</li>
        <li>Fix fencepost error in reverse cumulative histogram plotting.</li>
        <li>Fix missing secondary X axis bug in stacked Time plots.</li>
        <li>Fix bug in histogram plotting that could cause crashes
            for small ranges far from the origin.</li>
        <li>Fix aux axis/legend positioning issue in sky plot
            (regression bug introduced at v3.4-8).</li>
        <li>Fix rendering bug for
            <ref id="mode-gui"><code>omode=gui</code></ref>
            with huge (&gt;130 million row) tables.</li>
        <li>Fix <code>datalinklint</code> bug that reported
            a <code>W-SDND</code> warning for all tables having
            multiple service descriptors.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.4-10">Version 3.4-10 (29 February 2024)</dt>
<dd><p>
    <dl>
    <dt>New functionality:</dt>
    <dd><p>
        <ul>
        <li>Add support for HAPI time series services:
            <ref id="inHapi">input handler</ref> and
            <ref id="scheme-hapi">scheme</ref>.</li>
        <li>Parameter
            <ref id="plot2cube-geomN"><code>geomN</code></ref> in
            <ref id="plot2cube"><code>plot2cube</code></ref> can now take
            value "<code>vector</code>" to use <code>xyz</code>
            3-element array values as coordinate specifiers.</li>
        <li>The <ref id="layer-spectrogram">spectrogram</ref> plot layer
            now tries to plot spectra on a spectral axis,
            as controlled by the new
            <ref id="layer-spectrogram-scalespecN">scalespecN</ref>
            option.</li>
        <li>New <code>plot2time</code> parameter
            <ref id="plot2time-cellgap"><code>cellgap</code></ref>
            configures gap between stacked plots.</li>
        <li>New array functions
            <ref id="Arrays-loop"><code>loop</code></ref> with arguments
            <code>(start,end)</code> or <code>(start,end,step)</code>.</li>
        <li>The first argument of the <code>inMoc</code> and
            <code>nearMoc</code> <ref id="Coverage">Coverage</ref> functions
            can now be an ASCII MOC string as well as a MOC file location
            or VizieR table identifier.</li>
        <li>Add <code>-xtype</code> flag to filters
            <ref id="addcol"><code>addcol</code></ref>,
            <ref id="colmeta"><code>colmeta</code></ref>,
            <ref id="replacecol"><code>replacecol</code></ref> and
            <ref id="setparam"><code>setparam</code></ref>.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes:</dt>
    <dd><p>
        <ul>
        <li>Add a <ref id="quoting">Strings and Quoting</ref> section
            to this document.</li>
        <li>The <code>CoosysRefposition</code> metadata item (VOTable 1.5)
            is now listed if present by the <ref id="meta">meta</ref> filter.
            This information is also propagated to VOTable output if the
            output VOTable version is set to 1.5
            (currently not the default).</li>
        <li>Allow use of trailing backslash to specify multiline filters
            in '<code>@</code>'-indirection files for
            <ref id="tpipe-cmd"><code>cmd</code></ref>-type parameters.</li>
        <li>ISO-8601 conversion functions in <ref id="Times">Times</ref>
            now accept a trailing "<code>Z</code>" even for date-only
            specifications,
            and now accept <code>YYYY-DDD</code> format for dates
            as well as <code>YYYY-MM-DD</code>.</li>
        <li>Add new <ref id="Shaders">colour map</ref> Sunset.</li>
        <li><ref id="datalinklint"><code>datalinklint</code></ref>
            now checks and uses <code>INFO/@name="standardID"</code>
            version declarations in the links document being validated.
            DataLink handling and <code>datalinklint</code> are now believed
            to be up to date w.r.t. REC-DataLink-1.1.</li>
        <li>Some fixes/improvements to the <code>stilts</code> startup script.
            It now works better in the MacOS DMG image,
            it's more robust against pathnames with embedded spaces,
            and it will use <code>topcat-extra.jar</code> if present
            in preference to <code>topcat-full.jar</code>.</li>
        <li>Parameters
            <ref id="plot2sky-datasysN"><code>datasysN</code></ref>
            in <ref id="plot2sky"><code>plot2sky</code></ref> and
            <ref id="plot2cube-geomN"><code>geomN</code></ref>
            in <ref id="plot2cube"><code>plot2cube</code></ref>
            are now included in the documentation.
            The previously undocumented option to supply
            <code>plot2cube</code> coordinates in spherical polar form
            is now documented.</li>
        <li>Improve error reporting for corrupted/truncated FITS files.</li>
        <li>Use authentication and HTTP-level compression for
            external (href-referenced) VOTable STREAM data.</li>
        <li>Update VOTable 1.5 schema from WD to REC-VOTable-1.5
            (affects <code>votlint</code>).</li>
        <li>Upgrade Unity to v1.1, which corresponds to REC-VOUnits-1.1.
            Only minor functional differences expected since previous
            version (1.1-b1).</li>
        <li>Upgrade VOLLT adqlLib to official 2.0-beta release.</li>
        <li>Upgrade snakeyaml library (used for ECSV headers)
            from 1.25 to 2.2.
            No change in behaviour (or security) expected,
            but prevents vulnerability warnings in some circumstances.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes:</dt>
    <dd><p>
        <ul>
        <li>Fix sky match failure in case of very large error radius
            (&gt;22 deg).</li>
        <li>Fix <code>votlint</code> to permit empty <code>TD</code>
            elements in all cases for VOTable versions &gt;=1.3.
            Previously empty cells for integer and array fields
            provoked an error report
            (which is correct for VOTable 1.0-1.2, but not for 1.3+).</li>
        <li>Fix <ref id="tcat"><code>tcat</code></ref
              >/<ref id="tcatn"><code>tcatn</code></ref> bug that sometimes
            resulted in no rows included from an input table.</li>
        <li>Cope better with out of range pixel indices in
            <ref id="layer-healpix">healpix</ref> plots.</li>
        <li>Fix issue with <code>fat_triangle_up</code> and
            <code>fat_triangle_down</code>
            <ref id="layer-mark-shapeN">marker shape</ref> names.</li>
        <li>Fix FITS multi-table read bug that sometimes generated a
            non-fatal error message when reading basic FITS files
            using <ref id="tmulti"><code>tmulti</code></ref>.</li>
        <li>Some spectrogram bug fixes.</li>
        <li>Fix <ref id="plot2time"><code>plot2time</code></ref> so that
            zone-specific text style configuration works correctly for
            legends and colour ramp annotations.</li>
        <li>Update JSAMP to v1.3.8.
            This may improve hub Web Profile browser compatibility.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.5">Version 3.5 (7 August 2024)</dt>
<dd><p>
    <dl>
    <dt>Source code</dt>
    <dd><p>At this version the source code and build system have been
        substantially tidied up so that the application can be built
        straightforwardly using modern versions of Java (8, 11, 17, 21)
        with a minimum of warnings.
        This should be mostly invisible to users,
        but a few behaviour changes may be observed:
        <ul>
        <li>Some changes have been made to URL handling.
            Syntactically invalid URIs are now mostly rejected.
            This should not be noticeable in most cases,
            but questionable usages like embedded spaces in URLs may need
            to be replaced by their <code>%</code>-encoded equivalents.</li>
        </ul>
        </p></dd>
    <dt>New functionality</dt>
    <dd><p>
        <ul>
        <li>Commands <ref id="tapquery"><code>tapquery</code></ref>,
                     <ref id="tapskymatch"><code>tapskymatch</code></ref>,
                     <ref id="taplint"><code>taplint</code></ref> and
                     <ref id="tapresume"><code>tapresume</code></ref>
            have a new parameter <code>auth</code>;
            if true then an <ref id="AuthManager">authentication attempt</ref>
            will be made for TAP services that offer
            optional authentication.</li>
        <li>Arbitrary quantiles (e.g. <code>Q.95</code>, <code>Q.001</code>) 
            can now be specified as combiners in
            <ref id="tcube"><code>tcube</code></ref>,
            <ref id="tgridmap"><code>tgridmap</code></ref>,
            <ref id="tskymap"><code>tskymap</code></ref> and
            <ref id="tgroup"><code>tgroup</code></ref>
            alongside the predefined quartile specifiers
            (<code>Q1</code>, <code>median</code>, <code>Q3</code>).</li>
        <li>New matcher variants
            <ref id="ErrorCartesianMatchEngine"><code>nd_err_q</code></ref>
            and <ref id="SkyMatchEngine-err"><code>skyerr_q</code></ref>
            introduced for combining per-object errors in quadrature.</li>
        <li>New <ref id="scheme-test"><code>test</code> scheme</ref> options
            "<code>g</code>" and "<code>w</code>".</li>
        </ul>
        </p></dd>
    <dt>I/O handler improvements</dt>
    <dd><p>
        <ul>
        <li>Documentation for <ref id="ioFormats">I/O handler</ref> 
            config options now mostly reports their default values.</li>
        <li>Add to this document detailed documentation of
            <ref id="fitsPlus">FITS-plus</ref> and
            <ref id="wideFits">Wide FITS</ref> conventions.</li>
        <li>FITS output handling improved and reorganised to provide
            more flexible configuration options;
            this is now all documented under the <ref id="outFits">fits</ref>
            heading, there is no longer a separate <code>colfits</code>
            section in the documentation.</li>
        <li>New config option <code>date</code>
            for <ref id="outVotable">VOTable output handler</ref>
            to control whether a datestamp comment is written.</li>
        <li><ref id="outFits">Fits-plus</ref> output handlers
            now honour <code>date</code>
            config option for VOTable metadata
            as well as for <code>DATE-HDU</code> header card.</li>
        <li>Xtypes are now written into FITS headers using the
            non-standard header card <code>TXTYPnnn</code>.</li>
        <li>Non-standard FITS headers
            <code>TUCDnnn</code> and <code>TUTYPnnn</code>
            are now written with comment parts, space permitting.</li>
        <li>Upgrade parquet support libraries to parquet-mr 1.13.1.
            This means that Snappy compression is now supported in parquet for 
            MacOS ARM, as well as other, architectures.</li>
        <li>Add support for LZ4_RAW compression to Parquet I/O handlers.</li>
        <li>Add <code>compression</code> config option to
            <ref id="outParquet">Parquet output handler</ref>
            to control compression type.</li>
        <li>Add <code>usedict</code> config option to
            <ref id="outParquet">Parquet output handler</ref>
            to control dictionary use on output.</li>
        <li>Decrease size of parquet output files in most cases
            when writing NaNs and empty strings/arrays.</li>
        <li>New config option <code>tryUrl</code>
            in <ref id="inParquet">parquet input handler</ref>;
            this is set false by default to avoid cryptic error messsages
            when trying to open remote parquet files.</li>
        <li>Parquet input handler now copes with some additional variants of
            array-valued columns.</li>
        <li>New config option <code>useFloat</code>
            in <ref id="inMrt">MRT input handler</ref>;
            this is set false by default to avoid a rare bug
            that could read large values as infinite.</li>
        </ul>
        </p></dd>
    <dt>Minor enhancements and behaviour changes</dt>
    <dd><p>
        <ul>
        <li>Treatment of null values in the astrometric epoch propagation
            functions
            <ref id="Gaia-epochProp"><code>epochProp</code></ref> and
            <ref id="Gaia-epochPropErr"><code>epochPropErr</code></ref>
            has changed; null values for parallax and proper motion are
            now treated as if zero rather than invalidating the propagation.
            This (partly) follows behaviour of corresponding functionality
            in Gaia and VO ADQL propagation UDFs, and is more likely
            what you want to see.</li>
        <li><ref id="votlint"><code>votlint</code></ref> now includes
            column/param name and row index with reports in more cases.</li>
        <li>Avoid writing <code>arraysize="1"</code>
            in <code>FIELD</code>/<code>PARAM</code> elements
            of VOTables with version &gt;=1.3,
            in accordance with VOTable 1.3 Erratum #3.</li>
        <li>Avoid non-VOUnit units in output VOTables in some cases.</li>
        <li>Retire some taplint DALI-examples warnings related to
            old TOPCAT versions.</li>
        <li>Upgrade JSAMP to v1.3.9.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes</dt>
    <dd><p>
        <ul>
        <li>Bugfix update of CDS HEALPix library to v0.30.3.</li>
        <li>Fix authentication bug in <code>taplint</code>;
            avoid spurious <code>E-UWS-NFND</code> reports
            when validating authenticated services.</li>
        <li>Fix broken STC-S BOX parsing for
            <ref id="layer-area"><code>area</code></ref> plots.</li>
        <li>Fix VOTable output bug that wrote infinite floating point
            array elements to TABLEDATA as "<code>+/-Infinity</code>"
            rather than "<code>+/-Inf"</code>.</li>
        <li>Fix some <code>votlint</code> bugs that could cause failure
            for certain inputs.</li>
        <li>Fix taplint EPN stage bug that required
            <code>coverage</code> xtype to be
            "<code>MOC</code>" instead of "<code>moc</code>".</li>
        <li>Avoid stack overflow errors for long ASCII MOCs in
            <ref id="Coverage">Coverage</ref> functions.</li>
        <li>Fix authentication bug to do with redirects.</li>
        <li>Fix parquet input handler bug related to compression.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

<dt id="v3.5-1">Version 3.5-1 (6 November 2024)</dt>
<dd><p>
    <dl>
    <dt>Enhancements</dt>
    <dd><p>
        <ul>
        <li>STC-S encoding in
            <ref id="layer-area"><code>area</code></ref>-like plots
            now copes with UNIONs of all shapes, not just of POLYGONs.</li>
        <li>More units (seconds with SI prefixes) are now properly handled
            in VOTable <code>TIMESYS</code>-referencing fields.</li>
        <li>New <code><ref id="plot2plane-labelangle">labelangle</ref></code>
            configuration option in most plot types;
            allows angled labels that can accommodate more major ticks
            on crowded axes.
            <code>plot2cube</code>/<code>plot2sphere</code> (sometimes)
            and <code>plot2matrix</code> now use angled axis labels
            by default.</li>
        <li>Histogram-like plots
            (<ref id="layer-histogram"><code>histogram</code></ref>,
             <ref id="layer-kde"><code>kde</code></ref>,
             <ref id="layer-knn"><code>knn</code></ref>,
             <ref id="layer-densogram"><code>densogram</code></ref>,
             <ref id="layer-gaussian"><code>gaussian</code></ref>)
            have a new
            <code><ref id="layer-histogram-sidewaysN">sideways</ref></code>
            configuration option that can draw the quantity being assessed
            along the vertical instead of the horizontal axis.</li>
        <li>Cope with whitespace round input URLs better than previous
            version (regression).</li>
        <li>Upgrade JSON-java library to 20240303.
            Behaviour is not expected to change in interesting ways,
            but some vulnerabilities are addressed.</li>
        </ul>
        </p></dd>
    <dt>Bugfixes</dt>
    <dd><p>
        <ul>
        <li>Fix bug in TAP curl logging.</li>
        <li>Fix regression bug (since v3.4-9) in HTTP 3xx redirect handling
            that failed to cope with relative Location field values.</li>
        <li>Fix <code>taplint</code> bug that did not recognise
            ADQL 2.1 feature <code>features-adql-conditional</code>
            but did recognise <code>features-adql-bitwise</code>.</li>
        </ul>
        </p></dd>
    </dl>
    </p></dd>

</dl>
</p>
</subsect>

</sect>

</appendices>

</docbody>

</sun>
