// HTMLParser Library $Name: v1_6 $ - A java-based parser for HTML
// http://sourceforge.org/projects/htmlparser
// Copyright (C) 2003 Derrick Oswald
//
// Revision Control Information
//
// $Source: /cvsroot/htmlparser/htmlparser/src/org/htmlparser/parserapplications/SiteCapturer.java,v $
// $Author: derrickoswald $
// $Date: 2005/04/12 11:27:41 $
// $Revision: 1.9 $
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//

package org.htmlparser.parserapplications;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashSet;
import javax.swing.JFileChooser;
import javax.swing.JOptionPane;

import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.PrototypicalNodeFactory;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.BaseHrefTag;
import org.htmlparser.tags.FrameTag;
import org.htmlparser.tags.ImageTag;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.tags.MetaTag;
import org.htmlparser.util.EncodingChangeException;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

/**
 * Save a web site locally.
 * Illustrative program to save a web site contents locally.
 * It was created to demonstrate URL rewriting in it's simplest form.
 * It uses customized tags in the NodeFactory to alter the URLs.
 * This program has a number of limitations:
 * <ul>
 * <li>it doesn't capture forms, this would involve too many assumptions</li>
 * <li>it doesn't capture script references, so funky onMouseOver and other
 * non-static content will not be faithfully reproduced</li>
 * <li>it doesn't handle style sheets</li>
 * <li>it doesn't dig into attributes that might reference resources, so
 * for example, background images won't necessarily be captured</li>
 * <li>worst of all, it gets confused when a URL both has content and is
 * the prefix for other content,
 * i.e. http://whatever.com/top and http://whatever.com/top/sub.html both
 * yield content, since this cannot be faithfully replicated to a static
 * directory structure (this happens a lot with servlet based sites)</li>
 *</ul>
 */
public class SiteCapturer
{
    /**
     * The web site to capture.
     * This is used as the base URL in deciding whether to adjust a link
     * and whether to capture a page or not.
     */
    protected String mSource;

    /**
     * The local directory to capture to.
     * This is used as a base prefix for files saved locally.
     */
    protected String mTarget;

    /**
     * The list of pages to capture.
     * Links are added to this list as they are discovered, and removed in
     * sequential order (FIFO queue) leading to a breadth
     * first traversal of the web site space.
     */
    protected ArrayList mPages;

    /**
     * The set of pages already captured.
     * Used to avoid repeated acquisition of the same page.
     */
    protected HashSet mFinished;

    /**
     * The list of resources to copy.
     * Images and other resources are added to this list as they are discovered.
     */
    protected ArrayList mImages;

    /**
     * The set of resources already copied.
     * Used to avoid repeated acquisition of the same images and other resources.
     */
    protected HashSet mCopied;

    /**
     * The parser to use for processing.
     */
    protected Parser mParser;

    /**
     * If <code>true</code>, save resources locally too,
     * otherwise, leave resource links pointing to original page.
     */
    protected boolean mCaptureResources;

    /**
     * The filter to apply to the nodes retrieved.
     */
    protected NodeFilter mFilter;

    /**
     * Copy buffer size.
     * Resources are moved to disk in chunks this size or less.
     */
    protected final int TRANSFER_SIZE = 4096;

    /**
     * Create a web site capturer.
     */
    public SiteCapturer ()
    {
        PrototypicalNodeFactory factory;

        mSource = null;
        mTarget = null;
        mPages = new ArrayList ();
        mFinished = new HashSet ();
        mImages = new ArrayList ();
        mCopied = new HashSet ();
        mParser = new Parser ();
        factory = new PrototypicalNodeFactory ();
        factory.registerTag (new LocalLinkTag ());
        factory.registerTag (new LocalFrameTag ());
        factory.registerTag (new LocalBaseHrefTag ());
        factory.registerTag (new LocalImageTag ());
        mParser.setNodeFactory (factory);
        mCaptureResources = true;
        mFilter = null;
    }

    /**
     * Getter for property source.
     * @return Value of property source.
     */
    public String getSource ()
    {
        return (mSource);
    }
    
    /**
     * Setter for property source.
     * This is the base URL to capture. URL's that don't start with this prefix
     * are ignored (left as is), while the ones with this URL as a base are
     * re-homed to the local target.
     * @param source New value of property source.
     */
    public void setSource (String source)
    {
        if (source.endsWith ("/"))
            source = source.substring (0, source.length () - 1);
        mSource = source;
    }
    
    /**
     * Getter for property target.
     * @return Value of property target.
     */
    public String getTarget ()
    {
        return (mTarget);
    }
    
    /**
     * Setter for property target.
     * This is the local directory under which to save the site's pages.
     * @param target New value of property target.
     */
    public void setTarget (String target)
    {
        mTarget = target;
    }

    /**
     * Getter for property captureResources.
     * If <code>true</code>, the images and other resources referenced by
     * the site and within the base URL tree are also copied locally to the
     * target directory. If <code>false</code>, the image links are left 'as
     * is', still refering to the original site.
     * @return Value of property captureResources.
     */
    public boolean getCaptureResources ()
    {
        return (mCaptureResources);
    }
    
    /**
     * Setter for property captureResources.
     * @param capture New value of property captureResources.
     */
    public void setCaptureResources (boolean capture)
    {
        mCaptureResources = capture;
    }
    
    
    /** Getter for property filter.
     * @return Value of property filter.
     *
     */
    public NodeFilter getFilter ()
    {
        return (mFilter);
    }
    
    /** Setter for property filter.
     * @param filter New value of property filter.
     *
     */
    public void setFilter (NodeFilter filter)
    {
        mFilter = filter;
    }
    
    /**
     * Returns <code>true</code> if the link is one we are interested in.
     * @param link The link to be checked.
     * @return <code>true</code> if the link has the source URL as a prefix
     * and doesn't contain '?' or '#'; the former because we won't be able to
     * handle server side queries in the static target directory structure and
     * the latter because presumably the full page with that reference has
     * already been captured previously. This performs a case insensitive
     * comparison, which is cheating really, but it's cheap.
     */
    protected boolean isToBeCaptured (String link)
    {
        return (
            link.toLowerCase ().startsWith (getSource ().toLowerCase ())
            && (-1 == link.indexOf ("?"))
            && (-1 == link.indexOf ("#")));
    }

    /**
     * Returns <code>true</code> if the link contains text/html content.
     * @param link The URL to check for content type.
     * @return <code>true</code> if the HTTP header indicates the type is
     * "text/html".
     * @exception ParserException If the supplied URL can't be read from.
     */
    protected boolean isHtml (String link)
        throws
            ParserException
    {
        URL url;
        URLConnection connection;
        String type;
        boolean ret;

        ret = false;
        try
        {
            url = new URL (link);
            connection = url.openConnection ();
            type = connection.getContentType ();
            if (type == null)
                ret = false;
            else
                ret = type.startsWith ("text/html");
        }
        catch (Exception e)
        {
            throw new ParserException ("URL " + link + " has a problem", e);
        }
        
        return (ret);
    }

    /**
     * Converts a link to local.
     * A relative link can be used to construct both a URL and a file name.
     * Basically, the operation is to strip off the base url, if any,
     * and then prepend as many dot-dots as necessary to make
     * it relative to the current page.
     * A bit of a kludge handles the root page specially by calling it
     * index.html, even though that probably isn't it's real file name.
     * This isn't pretty, but it works for me.
     * @param link The link to make relative.
     * @param current The current page URL, or empty if it's an absolute URL
     * that needs to be converted.
     * @return The URL relative to the current page.
     */
    protected String makeLocalLink (String link, String current)
    {
        int i;
        int j;
        String ret;

        if (link.equals (getSource ()) || (!getSource ().endsWith ("/") && link.equals (getSource () + "/")))
            ret = "index.html"; // handle the root page specially
        else if (link.startsWith (getSource ())
                && (link.length () > getSource ().length ()))
            ret = link.substring (getSource ().length () + 1);
        else
            ret = link; // give up
            
        // make it relative to the current page by prepending "../" for
        // each '/' in the current local path
        if ((null != current)
            && link.startsWith (getSource ())
            && (current.length () > getSource ().length ()))
        {
            current = current.substring (getSource ().length () + 1);
            i = 0;
            while (-1 != (j = current.indexOf ('/', i)))
            {
                ret = "../" + ret;
                i = j + 1;
            }
        }

        return (ret);
    }

    /**
     * Unescape a URL to form a file name.
     * Very crude.
     * @param raw The escaped URI.
     * @return The native URI.
     */
    protected String decode (String raw)
    {
        int length;
        int start;
        int index;
        int value;
        StringBuffer ret;
        
        ret = new StringBuffer (raw.length ());

        length = raw.length ();
        start = 0;
        while (-1 != (index = raw.indexOf ('%', start)))
        {   // append the part up to the % sign
            ret.append (raw.substring (start, index));
            // there must be two hex digits after the percent sign
            if (index + 2 < length)
            {
                try
                {
                    value = Integer.parseInt (raw.substring (index + 1, index + 3), 16);
                    ret.append ((char)value);
                    start = index + 3;
                }
                catch (NumberFormatException nfe)
                {
                    ret.append ('%');
                    start = index + 1;
                }
            }
            else
            {   // this case is actually illegal in a URI, but...
                ret.append ('%');
                start = index + 1;
            }
        }
        ret.append (raw.substring (start));
        
        return (ret.toString ());
    }

    /**
     * Copy a resource (image) locally.
     * Removes one element from the 'to be copied' list and saves the
     * resource it points to locally as a file.
     */
    protected void copy ()
    {
        String link;
        String raw;
        String name;
        File file;
        File dir;
        URL source;
        byte[] data;
        InputStream in;
        FileOutputStream out;
        int read;

        link = (String)mImages.remove (0);
        mCopied.add (link);

        if (getCaptureResources ())
        {
            raw = makeLocalLink (link, "");
            name = decode (raw);
            file = new File (getTarget (), name);
            System.out.println ("copying " + link + " to " + file.getAbsolutePath ());
            // ensure directory exists
            dir = file.getParentFile ();
            if (!dir.exists ())
                dir.mkdirs ();
            try
            {
                source = new URL (link);
                data = new byte [TRANSFER_SIZE];
                try
                {
                    in = source.openStream ();
                    try
                    {
                        out = new FileOutputStream (file);
                        try
                        {
                            while (-1 != (read = in.read (data, 0, data.length)))
                                out.write (data, 0, read);
                        }
                        finally
                        {
                            out.close ();
                        }
                    }
                    catch (FileNotFoundException fnfe)
                    {
                        fnfe.printStackTrace ();
                    }
                    finally
                    {
                        in.close ();
                    }
                }
                catch (FileNotFoundException fnfe)
                {
                    System.err.println ("broken link " + fnfe.getMessage () + " ignored");
                }
            }
            catch (MalformedURLException murle)
            {
                murle.printStackTrace ();
            }
            catch (IOException ioe)
            {
                ioe.printStackTrace ();
            }
        }
    }
 
    /**
     * Process a single page.
     * @param filter The filter to apply to the collected nodes.
     * @exception ParserException If a parse error occurs.
     */
    protected void process (NodeFilter filter)
        throws
            ParserException
    {
        String url;
        int bookmark;
        NodeList list;
        NodeList robots;
        MetaTag robot;
        String content;
        File file;
        File dir;
        PrintWriter out;

        // get the next URL and add it to the done pile
        url = (String)mPages.remove (0);
        System.out.println ("processing " + url);
        mFinished.add (url);

        try
        {
            bookmark = mPages.size ();
            // fetch the page and gather the list of nodes
            mParser.setURL (url);
            try
            {
                list = new NodeList ();
                for (NodeIterator e = mParser.elements (); e.hasMoreNodes (); )
                    list.add (e.nextNode ()); // URL conversion occurs in the tags
            }
            catch (EncodingChangeException ece)
            {
                // fix bug #998195 SiteCatpurer just crashed
                // try again with the encoding now set correctly
                // hopefully mPages, mImages, mCopied and mFinished won't be corrupted
                mParser.reset ();
                list = new NodeList ();
                for (NodeIterator e = mParser.elements (); e.hasMoreNodes (); )
                    list.add (e.nextNode ());
            }

            // handle robots meta tag according to http://www.robotstxt.org/wc/meta-user.html
            // <meta name="robots" content="index,follow" />
            // <meta name="robots" content="noindex,nofollow" />
            robots = list.extractAllNodesThatMatch (
                new AndFilter (
                    new NodeClassFilter (MetaTag.class),
                    new HasAttributeFilter ("name", "robots")), true);
            if (0 != robots.size ())
            {
                robot = (MetaTag)robots.elementAt (0);
                content = robot.getAttribute ("content").toLowerCase ();
                if ((-1 != content.indexOf ("none")) || (-1 != content.indexOf ("nofollow")))
                    // reset mPages
                    for (int i = bookmark; i < mPages.size (); i++)
                        mPages.remove (i);
                if ((-1 != content.indexOf ("none")) || (-1 != content.indexOf ("noindex")))
                    return;
            }
    
            if (null != filter)
                list.keepAllNodesThatMatch (filter, true);

            // save the page locally
            file = new File (getTarget (), makeLocalLink (url, ""));
            dir = file.getParentFile ();
            if (!dir.exists ())
                dir.mkdirs ();
            else if (!dir.isDirectory ())
            {
                dir = new File (dir.getParentFile (), dir.getName () + ".content");
                if (!dir.exists ())
                    dir.mkdirs ();
                file = new File (dir, file.getName ());
            }
                
            try
            {
                out = new PrintWriter (new FileOutputStream (file));
                for (int i = 0; i < list.size (); i++)
                    out.print (list.elementAt (i).toHtml ());
                out.close ();
            }
            catch (FileNotFoundException fnfe)
            {
                fnfe.printStackTrace ();
            }
        }
        catch (ParserException pe)
        {
            String message;
            
            // this exception handling is suboptimal,
            // but it recognizes resources that aren't text/html
            message = pe.getMessage ();
            if ((null != message) && (message.endsWith ("does not contain text")))
            {
                if (!mCopied.contains (url))
                    if (!mImages.contains (url))
                        mImages.add (url);
                mFinished.remove (url);
            }
            else
                throw pe;
        }
    }

    /**
     * Link tag that rewrites the HREF.
     * The HREF is changed to a local target if it matches the source.
     */
    class LocalLinkTag extends LinkTag
    {
        public void doSemanticAction ()
            throws
                ParserException
        {
            boolean html;
            String link;

            // get the link
            link = getLink ();
            // check if it needs to be captured
            if (isToBeCaptured (link))
            {
                // add the link to a list to be processed
                if (mFinished.contains (link))
                    html = true;
                else if (mPages.contains (link))
                    html = true;
                else if (mCopied.contains (link))
                    html = false;
                else if (mImages.contains (link))
                    html = false;
                else
                {   // this test is expensive, do it reluctantly
                    html = isHtml (link);
                    if (html)
                        mPages.add (link);
                    else
                        mImages.add (link);
                }
                // alter the link
                if (html || (!html && getCaptureResources ()))
                    link = makeLocalLink (link, mParser.getLexer ().getPage ().getUrl ());
                setLink (link);
            }
        }
    }

    /**
     * Frame tag that rewrites the SRC URLs.
     * The SRC URLs are mapped to local targets if they match the source.
     */
    class LocalFrameTag extends FrameTag
    {
        public void doSemanticAction ()
            throws
                ParserException
        {
            boolean html;
            String link;

            // get the link
            link = getFrameLocation ();
            // check if it needs to be captured
            if (isToBeCaptured (link))
            {
                // add the link to a list to be processed
                if (mFinished.contains (link))
                    html = true;
                else if (mPages.contains (link))
                    html = true;
                else if (mCopied.contains (link))
                    html = false;
                else if (mImages.contains (link))
                    html = false;
                else
                {   // this test is expensive, do it reluctantly
                    html = isHtml (link);
                    if (html)
                        mPages.add (link);
                    else
                        mImages.add (link);
                }
                // alter the link
                if (html || (!html && getCaptureResources ()))
                    link = makeLocalLink (link, mParser.getLexer ().getPage ().getUrl ());
                setFrameLocation (link);
            }
        }
    }

    /**
     * Image tag that rewrites the SRC URL.
     * If resources are being captured the SRC is mapped to a local target if
     * it matches the source, otherwise it is convered to a full URL to point
     * back to the original site.
     */
    class LocalImageTag extends ImageTag
    {
        public void doSemanticAction ()
            throws
                ParserException
        {
            String image;
            
            // get the image url
            image = getImageURL ();
            // check if it needs to be captured
            if (isToBeCaptured (image))
            {   // add the image to the list needing to be copied
                if (!mCopied.contains (image))
                    if (!mImages.contains (image))
                        mImages.add (image);
                if (getCaptureResources ())
                    image = makeLocalLink (image, mParser.getLexer ().getPage ().getUrl ());
                // alter the link
                setImageURL (image);
            }
        }
    }

    /**
     * Base tag that doesn't show.
     * The toHtml() method is overridden to return an empty string,
     * effectively shutting off the base reference.
     */
    class LocalBaseHrefTag extends BaseHrefTag
    {
        // we don't want to have a base pointing back at the source page
        public String toHtml ()
        {
            return ("");
        }
    }

    /**
     * Perform the capture.
     */
    public void capture ()
    {
       
        mPages.clear ();
        mPages.add (getSource ());
        while (0 != mPages.size ())
            try
            {
                process (getFilter ());
                while (0 != mImages.size ())
                    copy ();
            }
            catch (ParserException pe)
            {   // this exception handling is suboptimal,
                // but it messages correctly about broken links
                Throwable throwable;
                
                throwable = pe.getThrowable ();
                if (null != throwable)
                {
                    throwable = throwable.getCause ();
                    if (throwable instanceof FileNotFoundException)
                        System.err.println ("broken link " + ((FileNotFoundException)throwable).getMessage () + " ignored");
                    else
                        pe.printStackTrace ();
                }
                else
                    pe.printStackTrace ();
            }
    }

    /**
     * Mainline to capture a web site locally.
     * @param args The command line arguments.
     * There are three arguments the web site to capture, the local directory
     * to save it to, and a flag (true or false) to indicate whether resources
     * such as images and video are to be captured as well.
     * These are requested via dialog boxes if not supplied.
     * @exception MalformedURLException If the supplied URL is invalid.
     * @exception IOException If an error occurs reading the page or resources.
     */
    public static void main (String[] args)
        throws
            MalformedURLException,
            IOException
    {
        SiteCapturer worker;
        String url;
        JFileChooser chooser;
        URL source;
        String path;
        File target;
        Boolean capture;
        int ret;
        
        worker = new SiteCapturer ();
        if (0 >= args.length)
        {
            url = (String)JOptionPane.showInputDialog (
                null,
                "Enter the URL to capture:",
                "Web Site",
                JOptionPane.PLAIN_MESSAGE,
                null,
                null,
                "http://htmlparser.sourceforge.net/wiki");
            if (null != url)
                worker.setSource (url);
            else
                System.exit (1);
        }
        else
            worker.setSource (args[0]);
        if (1 >= args.length)
        {
            url = worker.getSource ();
            source = new URL (url);
            path = new File (new File ("." + File.separator), source.getHost () + File.separator).getCanonicalPath ();
            target = new File (path);
            chooser = new JFileChooser (target);
            chooser.setDialogType (JFileChooser.SAVE_DIALOG);
            chooser.setFileSelectionMode (JFileChooser.DIRECTORIES_ONLY);
            chooser.setSelectedFile (target); // this doesn't frickin' work
            chooser.setMultiSelectionEnabled (false);
            chooser.setDialogTitle ("Target Directory");
            ret = chooser.showSaveDialog (null);
            if (ret == JFileChooser.APPROVE_OPTION)
                worker.setTarget (chooser.getSelectedFile ().getAbsolutePath ());
            else
                System.exit (1);
        }
        else
            worker.setTarget (args[1]);
        if (2 >= args.length)
        {
            capture = (Boolean)JOptionPane.showInputDialog (
                null,
                "Should resources be captured:",
                "Capture Resources",
                JOptionPane.PLAIN_MESSAGE,
                null,
                new Object[] { Boolean.TRUE, Boolean.FALSE},
                Boolean.TRUE);
            if (null != capture)
                worker.setCaptureResources (capture.booleanValue ());
            else
                System.exit (1);
        }
        else
            worker.setCaptureResources ((Boolean.valueOf (args[2]).booleanValue ()));
        worker.capture ();
        
        System.exit (0);
    }
}
