/*
 * Copyright 1998 Finn Bock.
 *
 * This program contains material copyrighted by:
 * Copyright (c) 1991-1995 by Stichting Mathematisch Centrum, Amsterdam,
 * The Netherlands.
 */

/* note about impl:
  instanceof vs. CPython type(.) is .
*/

package org.python.modules;

import java.math.BigInteger;
import java.util.Map;

import org.python.core.ClassDictInit;
import org.python.core.Py;
import org.python.core.PyBoolean;
import org.python.core.PyBuiltinCallable;
import org.python.core.PyClass;
import org.python.core.PyDictionary;
import org.python.core.PyException;
import org.python.core.PyFile;
import org.python.core.PyFloat;
import org.python.core.PyFunction;
import org.python.core.PyInstance;
import org.python.core.PyInteger;
import org.python.core.PyList;
import org.python.core.PyLong;
import org.python.core.PyModule;
import org.python.core.PyNone;
import org.python.core.PyObject;
import org.python.core.PyReflectedFunction;
import org.python.core.PySequence;
import org.python.core.PyString;
import org.python.core.PyStringMap;
import org.python.core.PyTuple;
import org.python.core.PyType;
import org.python.core.PyUnicode;
import org.python.core.__builtin__;
import org.python.core.codecs;
import org.python.core.exceptions;
import org.python.core.imp;
import org.python.util.Generic;

/**
 *
 * From the python documentation:
 * <p>
 * The <tt>cPickle.java</tt> module implements a basic but powerful algorithm
 * for ``pickling'' (a.k.a. serializing, marshalling or flattening) nearly
 * arbitrary Python objects.  This is the act of converting objects to a
 * stream of bytes (and back: ``unpickling'').
 * This is a more primitive notion than
 * persistency -- although <tt>cPickle.java</tt> reads and writes file
 * objects, it does not handle the issue of naming persistent objects, nor
 * the (even more complicated) area of concurrent access to persistent
 * objects.  The <tt>cPickle.java</tt> module can transform a complex object
 * into a byte stream and it can transform the byte stream into an object
 * with the same internal structure.  The most obvious thing to do with these
 * byte streams is to write them onto a file, but it is also conceivable
 * to send them across a network or store them in a database.  The module
 * <tt>shelve</tt> provides a simple interface to pickle and unpickle
 * objects on ``dbm''-style database files.
 * <P>
 * <b>Note:</b> The <tt>cPickle.java</tt> have the same interface as the
 * standard module <tt>pickle</tt>except that <tt>Pickler</tt> and
 * <tt>Unpickler</tt> are factory functions, not classes (so they cannot be
 * used as base classes for inheritance).
 * This limitation is similar for the original cPickle.c version.
 *
 * <P>
 * Unlike the built-in module <tt>marshal</tt>, <tt>cPickle.java</tt> handles
 * the following correctly:
 * <P>
 *
 * <UL><LI>recursive objects (objects containing references to themselves)
 *
 * <P>
 *
 * <LI>object sharing (references to the same object in different places)
 *
 * <P>
 *
 * <LI>user-defined classes and their instances
 *
 * <P>
 *
 * </UL>
 *
 * <P>
 * The data format used by <tt>cPickle.java</tt> is Python-specific.  This has
 * the advantage that there are no restrictions imposed by external
 * standards such as XDR (which can't represent pointer sharing); however
 * it means that non-Python programs may not be able to reconstruct
 * pickled Python objects.
 *
 * <P>
 * By default, the <tt>cPickle.java</tt> data format uses a printable ASCII
 * representation.  This is slightly more voluminous than a binary
 * representation.  The big advantage of using printable ASCII (and of
 * some other characteristics of <tt>cPickle.java</tt>'s representation) is
 * that for debugging or recovery purposes it is possible for a human to read
 * the pickled file with a standard text editor.
 *
 * <P>
 * A binary format, which is slightly more efficient, can be chosen by
 * specifying a nonzero (true) value for the <i>bin</i> argument to the
 * <tt>Pickler</tt> constructor or the <tt>dump()</tt> and <tt>dumps()</tt>
 * functions.  The binary format is not the default because of backwards
 * compatibility with the Python 1.4 pickle module.  In a future version,
 * the default may change to binary.
 *
 * <P>
 * The <tt>cPickle.java</tt> module doesn't handle code objects.
 * <P>
 * For the benefit of persistency modules written using <tt>cPickle.java</tt>,
 * it supports the notion of a reference to an object outside the pickled
 * data stream.  Such objects are referenced by a name, which is an
 * arbitrary string of printable ASCII characters.  The resolution of
 * such names is not defined by the <tt>cPickle.java</tt> module -- the
 * persistent object module will have to implement a method
 * <tt>persistent_load()</tt>.  To write references to persistent objects,
 * the persistent module must define a method <tt>persistent_id()</tt> which
 * returns either <tt>None</tt> or the persistent ID of the object.
 *
 * <P>
 * There are some restrictions on the pickling of class instances.
 *
 * <P>
 * First of all, the class must be defined at the top level in a module.
 * Furthermore, all its instance variables must be picklable.
 *
 * <P>
 *
 * <P>
 * When a pickled class instance is unpickled, its <tt>__init__()</tt> method
 * is normally <i>not</i> invoked.  <b>Note:</b> This is a deviation
 * from previous versions of this module; the change was introduced in
 * Python 1.5b2.  The reason for the change is that in many cases it is
 * desirable to have a constructor that requires arguments; it is a
 * (minor) nuisance to have to provide a <tt>__getinitargs__()</tt> method.
 *
 * <P>
 * If it is desirable that the <tt>__init__()</tt> method be called on
 * unpickling, a class can define a method <tt>__getinitargs__()</tt>,
 * which should return a <i>tuple</i> containing the arguments to be
 * passed to the class constructor (<tt>__init__()</tt>).  This method is
 * called at pickle time; the tuple it returns is incorporated in the
 * pickle for the instance.
 * <P>
 * Classes can further influence how their instances are pickled -- if the
 * class defines the method <tt>__getstate__()</tt>, it is called and the
 * return state is pickled as the contents for the instance, and if the class
 * defines the method <tt>__setstate__()</tt>, it is called with the
 * unpickled state.  (Note that these methods can also be used to
 * implement copying class instances.)  If there is no
 * <tt>__getstate__()</tt> method, the instance's <tt>__dict__</tt> is
 * pickled.  If there is no <tt>__setstate__()</tt> method, the pickled
 * object must be a dictionary and its items are assigned to the new
 * instance's dictionary.  (If a class defines both <tt>__getstate__()</tt>
 * and <tt>__setstate__()</tt>, the state object needn't be a dictionary
 * -- these methods can do what they want.)  This protocol is also used
 * by the shallow and deep copying operations defined in the <tt>copy</tt>
 * module.
 * <P>
 * Note that when class instances are pickled, their class's code and
 * data are not pickled along with them.  Only the instance data are
 * pickled.  This is done on purpose, so you can fix bugs in a class or
 * add methods and still load objects that were created with an earlier
 * version of the class.  If you plan to have long-lived objects that
 * will see many versions of a class, it may be worthwhile to put a version
 * number in the objects so that suitable conversions can be made by the
 * class's <tt>__setstate__()</tt> method.
 *
 * <P>
 * When a class itself is pickled, only its name is pickled -- the class
 * definition is not pickled, but re-imported by the unpickling process.
 * Therefore, the restriction that the class must be defined at the top
 * level in a module applies to pickled classes as well.
 *
 * <P>
 *
 * <P>
 * The interface can be summarized as follows.
 *
 * <P>
 * To pickle an object <tt>x</tt> onto a file <tt>f</tt>, open for writing:
 *
 * <P>
 * <dl><dd><pre>
 * p = pickle.Pickler(f)
 * p.dump(x)
 * </pre></dl>
 *
 * <P>
 * A shorthand for this is:
 *
 * <P>
 * <dl><dd><pre>
 * pickle.dump(x, f)
 * </pre></dl>
 *
 * <P>
 * To unpickle an object <tt>x</tt> from a file <tt>f</tt>, open for reading:
 *
 * <P>
 * <dl><dd><pre>
 * u = pickle.Unpickler(f)
 * x = u.load()
 * </pre></dl>
 *
 * <P>
 * A shorthand is:
 *
 * <P>
 * <dl><dd><pre>
 * x = pickle.load(f)
 * </pre></dl>
 *
 * <P>
 * The <tt>Pickler</tt> class only calls the method <tt>f.write()</tt> with a
 * string argument.  The <tt>Unpickler</tt> calls the methods
 * <tt>f.read()</tt> (with an integer argument) and <tt>f.readline()</tt>
 * (without argument), both returning a string.  It is explicitly allowed to
 * pass non-file objects here, as long as they have the right methods.
 *
 * <P>
 * The constructor for the <tt>Pickler</tt> class has an optional second
 * argument, <i>bin</i>.  If this is present and nonzero, the binary
 * pickle format is used; if it is zero or absent, the (less efficient,
 * but backwards compatible) text pickle format is used.  The
 * <tt>Unpickler</tt> class does not have an argument to distinguish
 * between binary and text pickle formats; it accepts either format.
 *
 * <P>
 * The following types can be pickled:
 *
 * <UL><LI><tt>None</tt>
 *
 * <P>
 *
 * <LI>integers, long integers, floating point numbers
 *
 * <P>
 *
 * <LI>strings
 *
 * <P>
 *
 * <LI>tuples, lists and dictionaries containing only picklable objects
 *
 * <P>
 *
 * <LI>classes that are defined at the top level in a module
 *
 * <P>
 *
 * <LI>instances of such classes whose <tt>__dict__</tt> or
 * <tt>__setstate__()</tt> is picklable
 *
 * <P>
 *
 * </UL>
 *
 * <P>
 * Attempts to pickle unpicklable objects will raise the
 * <tt>PicklingError</tt> exception; when this happens, an unspecified
 * number of bytes may have been written to the file.
 *
 * <P>
 * It is possible to make multiple calls to the <tt>dump()</tt> method of
 * the same <tt>Pickler</tt> instance.  These must then be matched to the
 * same number of calls to the <tt>load()</tt> method of the
 * corresponding <tt>Unpickler</tt> instance.  If the same object is
 * pickled by multiple <tt>dump()</tt> calls, the <tt>load()</tt> will all
 * yield references to the same object.  <i>Warning</i>: this is intended
 * for pickling multiple objects without intervening modifications to the
 * objects or their parts.  If you modify an object and then pickle it
 * again using the same <tt>Pickler</tt> instance, the object is not
 * pickled again -- a reference to it is pickled and the
 * <tt>Unpickler</tt> will return the old value, not the modified one.
 * (There are two problems here: (a) detecting changes, and (b)
 * marshalling a minimal set of changes.  I have no answers.  Garbage
 * Collection may also become a problem here.)
 *
 * <P>
 * Apart from the <tt>Pickler</tt> and <tt>Unpickler</tt> classes, the
 * module defines the following functions, and an exception:
 *
 * <P>
 * <dl><dt><b><tt>dump</tt></a></b> (<var>object, file</var><big>[</big><var>,
 *      bin</var><big>]</big>)
 * <dd>
 * Write a pickled representation of <i>obect</i> to the open file object
 * <i>file</i>.  This is equivalent to
 * "<tt>Pickler(<i>file</i>, <i>bin</i>).dump(<i>object</i>)</tt>".
 * If the optional <i>bin</i> argument is present and nonzero, the binary
 * pickle format is used; if it is zero or absent, the (less efficient)
 * text pickle format is used.
 * </dl>
 *
 * <P>
 * <dl><dt><b><tt>load</tt></a></b> (<var>file</var>)
 * <dd>
 * Read a pickled object from the open file object <i>file</i>.  This is
 * equivalent to "<tt>Unpickler(<i>file</i>).load()</tt>".
 * </dl>
 *
 * <P>
 * <dl><dt><b><tt>dumps</tt></a></b> (<var>object</var><big>[</big><var>,
 *     bin</var><big>]</big>)
 * <dd>
 * Return the pickled representation of the object as a string, instead
 * of writing it to a file.  If the optional <i>bin</i> argument is
 * present and nonzero, the binary pickle format is used; if it is zero
 * or absent, the (less efficient) text pickle format is used.
 * </dl>
 *
 * <P>
 * <dl><dt><b><tt>loads</tt></a></b> (<var>string</var>)
 * <dd>
 * Read a pickled object from a string instead of a file.  Characters in
 * the string past the pickled object's representation are ignored.
 * </dl>
 *
 * <P>
 * <dl><dt><b><a name="l2h-3763"><tt>PicklingError</tt></a></b>
 * <dd>
 * This exception is raised when an unpicklable object is passed to
 * <tt>Pickler.dump()</tt>.
 * </dl>
 *
 *
 * <p>
 * For the complete documentation on the pickle module, please see the
 * "Python Library Reference"
 * <p><hr><p>
 *
 * The module is based on both original pickle.py and the cPickle.c
 * version, except that all mistakes and errors are my own.
 * <p>
 * @author Finn Bock, bckfnn@pipmail.dknet.dk
 * @version cPickle.java,v 1.30 1999/05/15 17:40:12 fb Exp
 */
public class cPickle implements ClassDictInit {
    /**
     * The doc string
     */
    public static String __doc__ =
       "Java implementation and optimization of the Python pickle module\n";

    /**
     * The program version.
     */
    public static String __version__ = "1.30";

    /**
     * File format version we write.
     */
    public static final String format_version = "2.0";

    /**
     * Old format versions we can read.
     */
    public static final String[] compatible_formats =
                new String[] { "1.0", "1.1", "1.2", "1.3", "2.0" };

    /**
     * Highest protocol version supported.
     */
    public static final int HIGHEST_PROTOCOL = 2;

    public static String[] __depends__ = new String[] {
        "copy_reg",
    };

    public static PyObject PickleError;
    public static PyObject PicklingError;
    public static PyObject UnpickleableError;
    public static PyObject UnpicklingError;
    public static PyObject BadPickleGet;

    final static char MARK            = '(';
    final static char STOP            = '.';
    final static char POP             = '0';
    final static char POP_MARK        = '1';
    final static char DUP             = '2';
    final static char FLOAT           = 'F';
    final static char INT             = 'I';
    final static char BININT          = 'J';
    final static char BININT1         = 'K';
    final static char LONG            = 'L';
    final static char BININT2         = 'M';
    final static char NONE            = 'N';
    final static char PERSID          = 'P';
    final static char BINPERSID       = 'Q';
    final static char REDUCE          = 'R';
    final static char STRING          = 'S';
    final static char BINSTRING       = 'T';
    final static char SHORT_BINSTRING = 'U';
    final static char UNICODE         = 'V';
    final static char BINUNICODE      = 'X';
    final static char APPEND          = 'a';
    final static char BUILD           = 'b';
    final static char GLOBAL          = 'c';
    final static char DICT            = 'd';
    final static char EMPTY_DICT      = '}';
    final static char APPENDS         = 'e';
    final static char GET             = 'g';
    final static char BINGET          = 'h';
    final static char INST            = 'i';
    final static char LONG_BINGET     = 'j';
    final static char LIST            = 'l';
    final static char EMPTY_LIST      = ']';
    final static char OBJ             = 'o';
    final static char PUT             = 'p';
    final static char BINPUT          = 'q';
    final static char LONG_BINPUT     = 'r';
    final static char SETITEM         = 's';
    final static char TUPLE           = 't';
    final static char EMPTY_TUPLE     = ')';
    final static char SETITEMS        = 'u';
    final static char BINFLOAT        = 'G';

    final static char PROTO           = 0x80;
    final static char NEWOBJ          = 0x81;
    final static char EXT1            = 0x82;
    final static char EXT2            = 0x83;
    final static char EXT4            = 0x84;
    final static char TUPLE1          = 0x85;
    final static char TUPLE2          = 0x86;
    final static char TUPLE3          = 0x87;
    final static char NEWTRUE         = 0x88;
    final static char NEWFALSE        = 0x89;
    final static char LONG1           = 0x8A;
    final static char LONG4           = 0x8B;

    private static PyDictionary dispatch_table;
    private static PyDictionary extension_registry;
    private static PyDictionary inverted_registry;


    private static PyType BuiltinCallableType = PyType.fromClass(PyBuiltinCallable.class);

    private static PyType ReflectedFunctionType = PyType.fromClass(PyReflectedFunction.class);

    private static PyType ClassType = PyType.fromClass(PyClass.class);

    private static PyType TypeType = PyType.fromClass(PyType.class);

    private static PyType DictionaryType = PyType.fromClass(PyDictionary.class);

    private static PyType StringMapType = PyType.fromClass(PyStringMap.class);

    private static PyType FloatType = PyType.fromClass(PyFloat.class);

    private static PyType FunctionType = PyType.fromClass(PyFunction.class);

    private static PyType InstanceType = PyType.fromClass(PyInstance.class);

    private static PyType IntType = PyType.fromClass(PyInteger.class);

    private static PyType ListType = PyType.fromClass(PyList.class);

    private static PyType LongType = PyType.fromClass(PyLong.class);

    private static PyType NoneType = PyType.fromClass(PyNone.class);

    private static PyType StringType = PyType.fromClass(PyString.class);

    private static PyType UnicodeType = PyType.fromClass(PyUnicode.class);

    private static PyType TupleType = PyType.fromClass(PyTuple.class);

    private static PyType FileType = PyType.fromClass(PyFile.class);

    private static PyType BoolType = PyType.fromClass(PyBoolean.class);


    private static PyObject dict;

    private static final int BATCHSIZE = 1024;

    /**
     * Initialization when module is imported.
     */
    public static void classDictInit(PyObject dict) {
        cPickle.dict = dict;

        // XXX: Hack for JPython 1.0.1. By default __builtin__ is not in
        // sys.modules.
        imp.importName("__builtin__", true);

        PyModule copyreg = (PyModule)importModule("copy_reg");

        dispatch_table = (PyDictionary)copyreg.__getattr__("dispatch_table");
        extension_registry = (PyDictionary)copyreg.__getattr__("_extension_registry");
        inverted_registry = (PyDictionary)copyreg.__getattr__("_inverted_registry");

        PickleError = Py.makeClass("PickleError", Py.Exception, _PickleError());
        PicklingError = Py.makeClass("PicklingError", PickleError, exceptionNamespace());
        UnpickleableError = Py.makeClass("UnpickleableError", PicklingError, _UnpickleableError());
        UnpicklingError = Py.makeClass("UnpicklingError", PickleError, exceptionNamespace());
        BadPickleGet = Py.makeClass("BadPickleGet", UnpicklingError, exceptionNamespace());
    }

    public static PyObject exceptionNamespace() {
        PyObject dict = new PyStringMap();
        dict.__setitem__("__module__", new PyString("cPickle"));
        return dict;
    }

    public static PyObject _PickleError() {
        dict = exceptionNamespace();
        dict.__setitem__("__str__", getJavaFunc("__str__", "_PickleError__str__"));
        return dict;
    }

    public static PyString _PickleError__str__(PyObject self, PyObject[] args, String[] kwargs) {
        PyObject selfArgs = self.__getattr__("args");
        if (selfArgs.__len__() > 0 && selfArgs.__getitem__(0).__len__()  > 0) {
            return selfArgs.__getitem__(0).__str__();
        } else {
            return new PyString("(what)");
        }
    }

    public static PyObject _UnpickleableError() {
        dict = exceptionNamespace();
        dict.__setitem__("__str__", getJavaFunc("__str__", "_UnpickleableError__str__"));
        return dict;
    }

    public static PyString _UnpickleableError__str__(PyObject self, PyObject[] args,
                                                     String[] kwargs) {
        PyObject selfArgs = self.__getattr__("args");
        PyObject a = selfArgs.__len__() > 0 ? selfArgs.__getitem__(0) : new PyString("(what)");
        return new PyString("Cannot pickle %s objects").__mod__(a).__str__();
    }

    /**
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>write</i> method. The data will be written as text.
     * @return a new Pickler instance.
     */
    public static Pickler Pickler(PyObject file) {
        return new Pickler(file, 0);
    }


    /**
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>write</i> method.
     * @param protocol  pickle protocol version (0 - text, 1 - pre-2.3 binary, 2 - 2.3)
     * @return         a new Pickler instance.
     */
    public static Pickler Pickler(PyObject file, int protocol) {
        return new Pickler(file, protocol);
    }


    /**
     * Returns a unpickler instance.
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>read</i> and <i>readline</i> method.
     * @return         a new Unpickler instance.
     */
    public static Unpickler Unpickler(PyObject file) {
        return new Unpickler(file);
    }


    /**
     * Shorthand function which pickles the object on the file.
     * @param object    a data object which should be pickled.
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>write</i>  method. The data will be written as
     *                  text.
     */
    public static void dump(PyObject object, PyObject file) {
        dump(object, file, 0);
    }

    /**
     * Shorthand function which pickles the object on the file.
     * @param object    a data object which should be pickled.
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>write</i> method.
     * @param protocol  pickle protocol version (0 - text, 1 - pre-2.3 binary, 2 - 2.3)
     */
    public static void dump(PyObject object, PyObject file, int protocol) {
        new Pickler(file, protocol).dump(object);
    }


    /**
     * Shorthand function which pickles and returns the string representation.
     * @param object    a data object which should be pickled.
     * @return         a string representing the pickled object.
     */
    public static PyString dumps(PyObject object) {
        return dumps(object, 0);
    }


    /**
     * Shorthand function which pickles and returns the string representation.
     * @param object    a data object which should be pickled.
     * @param protocol  pickle protocol version (0 - text, 1 - pre-2.3 binary, 2 - 2.3)
     * @return         a string representing the pickled object.
     */
    public static PyString dumps(PyObject object, int protocol) {
        cStringIO.StringIO file = cStringIO.StringIO();
        dump(object, file, protocol);
        return file.getvalue();
    }


    /**
     * Shorthand function which unpickles a object from the file and returns
     * the new object.
     * @param file      a file-like object, can be a cStringIO.StringIO,
     *                  a PyFile or any python object which implements a
     *                  <i>read</i> and <i>readline</i> method.
     * @return         a new object.
     */
    public static Object load(PyObject file) {
        return new Unpickler(file).load();
    }


    /**
     * Shorthand function which unpickles a object from the string and
     * returns the new object.
     * @param str       a strings which must contain a pickled object
     *                  representation.
     * @return         a new object.
     */
    public static Object loads(PyObject str) {
        cStringIO.StringIO file = cStringIO.StringIO(str.toString());
        return new Unpickler(file).load();
    }



    // Factory for creating PyIOFile representation.
    
    /**
     * The Pickler object
     * @see cPickle#Pickler(PyObject)
     * @see cPickle#Pickler(PyObject,int)
     */
    static public class Pickler {
        private PyIOFile file;
        private int protocol;

        /**
         * The undocumented attribute fast of the C version of cPickle disables
         * memoization. Since having memoization on won't break anything, having
         * this dummy setter for fast here won't break any code expecting it to
         * do something. However without it code that sets fast fails(ie
         * test_cpickle.py), so it's worth having.
         */
         public boolean fast = false;

        private PickleMemo memo = new PickleMemo();

        /**
         * To write references to persistent objects, the persistent module
         * must assign a method to persistent_id which returns either None
         * or the persistent ID of the object.
         * For the benefit of persistency modules written using pickle,
         * it supports the notion of a reference to an object outside
         * the pickled data stream.
         * Such objects are referenced by a name, which is an arbitrary
         * string of printable ASCII characters.
         */
        public PyObject persistent_id = null;

        /**
         * Hmm, not documented, perhaps it shouldn't be public? XXX: fixme.
         */
        public PyObject inst_persistent_id = null;


        public Pickler(PyObject file, int protocol) {
            this.file = PyIOFileFactory.createIOFile(file);
            this.protocol = protocol;
        }


        /**
         * Write a pickled representation of the object.
         * @param object        The object which will be pickled.
         */
        public void dump(PyObject object) {
            if (protocol >= 2) {
                file.write(PROTO);
                file.write((char) protocol);
            }
            save(object);
            file.write(STOP);
            file.flush();
        }

        private static final int get_id(PyObject o) {
            // we don't pickle Java instances so we don't have to consider that case
            return System.identityHashCode(o);
        }


        // Save name as in pickle.py but semantics are slightly changed.
        private void put(int i) {
            if (protocol > 0) {
                if (i < 256) {
                    file.write(BINPUT);
                    file.write((char)i);
                    return;
                }
                file.write(LONG_BINPUT);
                file.write((char)( i         & 0xFF));
                file.write((char)((i >>>  8) & 0xFF));
                file.write((char)((i >>> 16) & 0xFF));
                file.write((char)((i >>> 24) & 0xFF));
                return;
            }
            file.write(PUT);
            file.write(String.valueOf(i));
            file.write("\n");
        }


        // Same name as in pickle.py but semantics are slightly changed.
        private void get(int i) {
            if (protocol > 0) {
                if (i < 256) {
                    file.write(BINGET);
                    file.write((char)i);
                    return;
                }
                file.write(LONG_BINGET);
                file.write((char)( i         & 0xFF));
                file.write((char)((i >>>  8) & 0xFF));
                file.write((char)((i >>> 16) & 0xFF));
                file.write((char)((i >>> 24) & 0xFF));
                return;
            }
            file.write(GET);
            file.write(String.valueOf(i));
            file.write("\n");
        }


        private void save(PyObject object) {
            save(object, false);
        }


        private void save(PyObject object, boolean pers_save) {
            if (!pers_save && persistent_id != null && save_pers(object, persistent_id)) {
                return;
            }

            int d = get_id(object);

            PyType t = object.getType();

            if (t == TupleType && object.__len__() == 0) {
                if (protocol > 0)
                    save_empty_tuple(object);
                else
                    save_tuple(object);
                return;
            }

            int m = getMemoPosition(d, object);
            if (m >= 0) {
                get(m);
                return;
            }

            if (save_type(object, t))
                return;

            if (!pers_save && inst_persistent_id != null && save_pers(object, inst_persistent_id)) {
                return;
            }

            if (Py.isSubClass(t, PyType.TYPE)) {
                save_global(object);
                return;
            }

            PyObject tup = null;
            PyObject reduce = dispatch_table.__finditem__(t);
            if (reduce == null) {
                reduce = object.__findattr__("__reduce_ex__");
                if (reduce != null) {
                    tup = reduce.__call__(Py.newInteger(protocol));
                } else {
                    reduce = object.__findattr__("__reduce__");
                    if (reduce == null)
                        throw new PyException(UnpickleableError, object);
                    tup = reduce.__call__();
                }
            } else {
                tup = reduce.__call__(object);
            }

            if (tup instanceof PyString) {
                save_global(object, tup);
                return;
            }

            if (!(tup instanceof PyTuple)) {
                throw new PyException(PicklingError,
                            "Value returned by " + reduce.__repr__() +
                            " must be a tuple");
            }

            int l = tup.__len__();
            if (l < 2 || l > 5) {
                throw new PyException(PicklingError,
                            "tuple returned by " + reduce.__repr__() +
                            " must contain two to five elements");
            }

            PyObject callable = tup.__finditem__(0);
            PyObject arg_tup = tup.__finditem__(1);
            PyObject state = (l > 2) ? tup.__finditem__(2) : Py.None;
            PyObject listitems = (l > 3) ? tup.__finditem__(3) : Py.None;
            PyObject dictitems = (l > 4) ? tup.__finditem__(4) : Py.None;

            if (!(arg_tup instanceof PyTuple) && arg_tup != Py.None) {
                throw new PyException(PicklingError,
                            "Second element of tupe returned by " +
                            reduce.__repr__() + " must be a tuple");
            }
            save_reduce(callable, arg_tup, state, listitems, dictitems, object);
        }


        final private boolean save_pers(PyObject object, PyObject pers_func) {
            PyObject pid = pers_func.__call__(object);
            if (pid == Py.None) {
                return false;
            }

            if (protocol == 0) {
                if (!Py.isInstance(pid, PyString.TYPE)) {
                    throw new PyException(PicklingError, "persistent id must be string");
                }
                file.write(PERSID);
                file.write(pid.toString());
                file.write("\n");
            } else {
                save(pid, true);
                file.write(BINPERSID);
            }
            return true;
        }

        final private void save_reduce(PyObject callable, PyObject arg_tup,
                                       PyObject state, PyObject listitems, PyObject dictitems,
                                       PyObject object)
        {
            PyObject callableName = callable.__findattr__("__name__");
            if(protocol >= 2 && callableName != null
                    && "__newobj__".equals(callableName.toString())) {
                PyObject cls = arg_tup.__finditem__(0);
                if(cls.__findattr__("__new__") == null)
                    throw new PyException(PicklingError,
                                          "args[0] from __newobj__ args has no __new__");
                // TODO: check class
                save(cls);
                save(arg_tup.__getslice__(Py.One, Py.None));
                file.write(NEWOBJ);
            } else {
                save(callable);
                save(arg_tup);
                file.write(REDUCE);
            }

            // Memoize
            put(putMemo(get_id(object), object));

            if (listitems != Py.None) {
                batch_appends(listitems);
            }
            if (dictitems != Py.None) {
                batch_setitems(dictitems);
            }
            if (state != Py.None) {
                save(state);
                file.write(BUILD);
            }
        }



        final private boolean save_type(PyObject object, PyType type) {
            //System.out.println("save_type " + object + " " + cls);
            if (type == NoneType)
                save_none(object);
            else if (type == StringType)
                save_string(object);
            else if (type == UnicodeType)
                save_unicode(object);
            else if (type == IntType)
                save_int(object);
            else if (type == LongType)
                save_long(object);
            else if (type == FloatType)
                save_float(object);
            else if (type == TupleType)
                save_tuple(object);
            else if (type == ListType)
                save_list(object);
            else if (type == DictionaryType || type == StringMapType)
                save_dict(object);
            else if (type == InstanceType)
                save_inst((PyInstance)object);
            else if (type == ClassType)
                save_global(object);
            else if (type == TypeType)
                save_global(object);
            else if (type == FunctionType)
                save_global(object);
            else if (type == BuiltinCallableType)
                save_global(object);
            else if (type == ReflectedFunctionType)
                save_global(object);
            else if (type == BoolType)
                save_bool(object);
            else
                return false;
            return true;
        }



        final private void save_none(PyObject object) {
            file.write(NONE);
        }

        final private void save_int(PyObject object) {
            if (protocol > 0) {
                int l = ((PyInteger)object).getValue();
                char i1 = (char)( l         & 0xFF);
                char i2 = (char)((l >>> 8 ) & 0xFF);
                char i3 = (char)((l >>> 16) & 0xFF);
                char i4 = (char)((l >>> 24) & 0xFF);

                if (i3 == '\0' && i4 == '\0') {
                    if (i2 == '\0') {
                        file.write(BININT1);
                        file.write(i1);
                        return;
                    }
                    file.write(BININT2);
                    file.write(i1);
                    file.write(i2);
                    return;
                }
                file.write(BININT);
                file.write(i1);
                file.write(i2);
                file.write(i3);
                file.write(i4);
            } else {
                file.write(INT);
                file.write(object.toString());
                file.write("\n");
            }
        }

        private void save_bool(PyObject object) {
            int value = ((PyBoolean)object).getValue();
            if(protocol >= 2) {
                file.write(value != 0 ? NEWTRUE : NEWFALSE);
            } else {
                file.write(INT);
                file.write(value != 0 ? "01" : "00");
                file.write("\n");
            }
        }

        private void save_long(PyObject object) {
            if(protocol >= 2) {
                BigInteger integer = ((PyLong)object).getValue();

                if (integer.compareTo(BigInteger.ZERO) == 0) {
                    // It's 0 -- an empty bytestring.
                    file.write(LONG1);
                    file.write((char)0);
                    return;
                }

                byte[] bytes = integer.toByteArray();
                int l = bytes.length;
                if (l < 256) {
                    file.write(LONG1);
                    file.write((char)l);
                } else {
                    file.write(LONG4);
                    writeInt4(l);
                }
                // Write in reverse order: pickle orders by little
                // endian whereas BigInteger orders by big endian
                for (int i = l - 1; i >= 0; i--) {
                    int b = bytes[i] & 0xff;
                    file.write((char)b);
                }
            } else {
                file.write(LONG);
                file.write(object.toString());
                file.write("\n");
            }
        }

        private void writeInt4(int l) {
            char i1 = (char)( l         & 0xFF);
            char i2 = (char)((l >>> 8 ) & 0xFF);
            char i3 = (char)((l >>> 16) & 0xFF);
            char i4 = (char)((l >>> 24) & 0xFF);
            file.write(i1);
            file.write(i2);
            file.write(i3);
            file.write(i4);
        }


        final private void save_float(PyObject object) {
            if (protocol > 0) {
                file.write(BINFLOAT);
                double value= ((PyFloat) object).getValue();
                // It seems that struct.pack('>d', ..) and doubleToLongBits
                // are the same. Good for me :-)
                long bits = Double.doubleToLongBits(value);
                file.write((char)((bits >>> 56) & 0xFF));
                file.write((char)((bits >>> 48) & 0xFF));
                file.write((char)((bits >>> 40) & 0xFF));
                file.write((char)((bits >>> 32) & 0xFF));
                file.write((char)((bits >>> 24) & 0xFF));
                file.write((char)((bits >>> 16) & 0xFF));
                file.write((char)((bits >>>  8) & 0xFF));
                file.write((char)((bits >>>  0) & 0xFF));
            } else {
                file.write(FLOAT);
                file.write(object.toString());
                file.write("\n");
            }
        }


        final private void save_string(PyObject object) {
            String str = object.toString();

            if (protocol > 0) {
                int l = str.length();
                if (l < 256) {
                    file.write(SHORT_BINSTRING);
                    file.write((char)l);
                } else {
                    file.write(BINSTRING);
                    file.write((char)( l         & 0xFF));
                    file.write((char)((l >>> 8 ) & 0xFF));
                    file.write((char)((l >>> 16) & 0xFF));
                    file.write((char)((l >>> 24) & 0xFF));
                }
                file.write(str);
            } else {
                file.write(STRING);
                file.write(object.__repr__().toString());
                file.write("\n");
            }
            put(putMemo(get_id(object), object));
        }

        private void save_unicode(PyObject object) {
            if (protocol > 0) {
                String str = codecs.PyUnicode_EncodeUTF8(object.toString(), "struct");
                file.write(BINUNICODE);
                writeInt4(str.length());
                file.write(str);
            } else {
                file.write(UNICODE);
                file.write(codecs.PyUnicode_EncodeRawUnicodeEscape(object.toString(),
                                                        "strict", true));
                file.write("\n");
            }
            put(putMemo(get_id(object), object));
        }

        private void save_tuple(PyObject object) {
            int d = get_id(object);

            int len = object.__len__();

            if (len > 0 && len <= 3 && protocol >= 2) {
                for (int i = 0; i < len; i++)
                    save(object.__finditem__(i));
                int m = getMemoPosition(d, object);
                if (m >= 0) {
                    for (int i = 0; i < len; i++)
                        file.write(POP);
                    get(m);
                }
                else {
                    char opcode = (char) (TUPLE1 + len - 1);
                    file.write(opcode);
                    put(putMemo(d, object));
                }
                return;
            }

            file.write(MARK);

            for (int i = 0; i < len; i++)
                save(object.__finditem__(i));

            if (len > 0) {
                int m = getMemoPosition(d, object);
                if (m >= 0) {
                    if (protocol > 0) {
                        file.write(POP_MARK);
                        get(m);
                        return;
                    }
                    for (int i = 0; i < len+1; i++)
                        file.write(POP);
                    get(m);
                    return;
                }
            }
            file.write(TUPLE);
            put(putMemo(d, object));
        }


        final private void save_empty_tuple(PyObject object) {
            file.write(EMPTY_TUPLE);
        }

        private void save_list(PyObject object) {
            if (protocol > 0)
                file.write(EMPTY_LIST);
            else {
                file.write(MARK);
                file.write(LIST);
            }

            put(putMemo(get_id(object), object));

            batch_appends(object);
        }

        private void batch_appends(PyObject object) {
            int countInBatch = 0;
            for (PyObject nextObj : object.asIterable()) {
                if(protocol == 0) {
                    save(nextObj);
                    file.write(APPEND);
                } else {
                    if(countInBatch == 0) {
                        file.write(MARK);
                    }
                    countInBatch++;
                    save(nextObj);
                    if(countInBatch == BATCHSIZE) {
                        file.write(APPENDS);
                        countInBatch = 0;
                    }
                }
            }
            if (countInBatch > 0)
                file.write(APPENDS);
        }


        private void save_dict(PyObject object) {
            if (protocol > 0)
                file.write(EMPTY_DICT);
            else {
                file.write(MARK);
                file.write(DICT);
            }

            put(putMemo(get_id(object), object));

            batch_setitems(object.invoke("iteritems"));
        }

        private void batch_setitems(PyObject object) {
            if (protocol == 0) {
                // SETITEMS isn't available; do one at a time.
                for (PyObject p : object.asIterable()) {
                    if (!(p instanceof PyTuple) || p.__len__() != 2) {
                        throw Py.TypeError("dict items iterator must return 2-tuples");
                    }
                    save(p.__getitem__(0));
                    save(p.__getitem__(1));
                    file.write(SETITEM);
                }
            } else {
                // proto > 0:  write in batches of BATCHSIZE.
                PyObject obj;
                PyObject[] slice = new PyObject[BATCHSIZE];
                int n;
                do {
                    // Get next group of (no more than) BATCHSIZE elements.
                    for (n = 0; n < BATCHSIZE; n++) {
                        obj = object.__iternext__();
                        if (obj == null) {
                            break;
                        }
                        slice[n] = obj;
                    }

                    if (n > 1) {
                        // Pump out MARK, slice[0:n], APPENDS.
                        file.write(MARK);
                        for (int i = 0; i < n; i++) {
                            obj = slice[i];
                            save(obj.__getitem__(0));
                            save(obj.__getitem__(1));
                        }
                        file.write(SETITEMS);
                    } else if (n == 1) {
                        obj = slice[0];
                        save(obj.__getitem__(0));
                        save(obj.__getitem__(1));
                        file.write(SETITEM);
                    }
                } while (n == BATCHSIZE);
            }
        }


        final private void save_inst(PyInstance object) {
            PyClass cls = object.instclass;

            PySequence args = null;
            PyObject getinitargs = object.__findattr__("__getinitargs__");
            if (getinitargs != null) {
                args = (PySequence)getinitargs.__call__();
                // XXX Assert it's a sequence
                keep_alive(args);
            }

            file.write(MARK);
            if (protocol > 0)
                save(cls);

            if (args != null) {
                int len = args.__len__();
                for (int i = 0; i < len; i++)
                    save(args.__finditem__(i));
            }

            int mid = putMemo(get_id(object), object);
            if (protocol > 0) {
                file.write(OBJ);
                put(mid);
            } else {
                file.write(INST);
                file.write(cls.__findattr__("__module__").toString());
                file.write("\n");
                file.write(cls.__name__);
                file.write("\n");
                put(mid);
            }

            PyObject stuff = null;
            PyObject getstate = object.__findattr__("__getstate__");
            if (getstate == null) {
                stuff = object.__dict__;
            } else {
                stuff = getstate.__call__();
                keep_alive(stuff);
            }
            save(stuff);
            file.write(BUILD);
        }


        final private void save_global(PyObject object) {
            save_global(object, null);
        }


        final private void save_global(PyObject object, PyObject name) {
            if (name == null)
                name = object.__findattr__("__name__");

            PyObject module = object.__findattr__("__module__");
            if (module == null || module == Py.None)
                module = whichmodule(object, name);

            if(protocol >= 2) {
                PyTuple extKey = new PyTuple(module, name);
                PyObject extCode = extension_registry.get(extKey);
                if(extCode != Py.None) {
                    int code = ((PyInteger)extCode).getValue();
                    if(code <= 0xFF) {
                        file.write(EXT1);
                        file.write((char)code);
                    } else if(code <= 0xFFFF) {
                        file.write(EXT2);
                        file.write((char)(code & 0xFF));
                        file.write((char)(code >> 8));
                    } else {
                        file.write(EXT4);
                        writeInt4(code);
                    }
                    return;
                }
            }

            file.write(GLOBAL);
            file.write(module.toString());
            file.write("\n");
            file.write(name.toString());
            file.write("\n");
            put(putMemo(get_id(object), object));
        }


        final private int getMemoPosition(int id, Object o) {
            return memo.findPosition(id, o);
        }

        final private int putMemo(int id, PyObject object) {
            int memo_len = memo.size() + 1;
            memo.put(id, memo_len, object);
            return memo_len;
        }


        /**
         * Keeps a reference to the object x in the memo.
         *
         * Because we remember objects by their id, we have
         * to assure that possibly temporary objects are kept
         * alive by referencing them.
         * We store a reference at the id of the memo, which should
         * normally not be used unless someone tries to deepcopy
         * the memo itself...
         */
        final private void keep_alive(PyObject obj) {
            int id = System.identityHashCode(memo);
            PyList list = (PyList) memo.findValue(id, memo);
            if (list == null) {
                list = new PyList();
                memo.put(id, -1, list);
            }
            list.append(obj);
        }

    }

    private static Map<PyObject,PyObject> classmap = Generic.map();

    final private static PyObject whichmodule(PyObject cls,
                                              PyObject clsname)
    {
        PyObject name = classmap.get(cls);
        if (name != null)
            return name;

        name = new PyString("__main__");

        // For use with JPython1.0.x
        //PyObject modules = sys.modules;

        // For use with JPython1.1.x
        //PyObject modules = Py.getSystemState().modules;

        PyObject sys = imp.importName("sys", true);
        PyObject modules = sys.__findattr__("modules");
        PyObject keylist = modules.invoke("keys");

        int len = keylist.__len__();
        for (int i = 0; i < len; i++) {
            PyObject key = keylist.__finditem__(i);
            PyObject value = modules.__finditem__(key);

            if (!key.equals("__main__") &&
                    value.__findattr__(clsname.toString().intern()) == cls) {
                name = key;
                break;
            }
        }

        classmap.put(cls, name);
        //System.out.println(name);
        return name;
    }


    /*
     * A very specialized and simplified version of PyStringMap. It can
     * only use integers as keys and stores both an integer and an object
     * as value. It is very private! And should only be used thread-confined.
     */
    static private class PickleMemo {
        //Table of primes to cycle through
        private final int[] primes = {
            13, 61, 251, 1021, 4093,
            5987, 9551, 15683, 19609, 31397,
            65521, 131071, 262139, 524287, 1048573, 2097143,
            4194301, 8388593, 16777213, 33554393, 67108859,
            134217689, 268435399, 536870909, 1073741789,};

        private transient int[] keys;
        private transient int[] position;
        private transient Object[] values;

        private int size;
        private transient int filled;
        private transient int prime;

        public PickleMemo(int capacity) {
            prime = 0;
            keys = null;
            values = null;
            resize(capacity);
        }

        public PickleMemo() {
            this(4);
        }

        public int size() {
            return size;
        }

        private int findIndex(int key, Object value) {
            int[] table = keys;
            int maxindex = table.length;
            int index = (key & 0x7fffffff) % maxindex;

            // Fairly aribtrary choice for stepsize...
            int stepsize = maxindex / 5;

            // Cycle through possible positions for the key;
            //int collisions = 0;
            while (true) {
                int tkey = table[index];
                if (tkey == key && value == values[index]) {
                    return index;
                }
                if (values[index] == null) return -1;
                index = (index+stepsize) % maxindex;
            }
        }

        public int findPosition(int key, Object value) {
            int idx = findIndex(key, value);
            if (idx < 0) return -1;
            return position[idx];
        }


        public Object findValue(int key, Object value) {
            int idx = findIndex(key, value);
            if (idx < 0) return null;
            return values[idx];
        }


        private final void insertkey(int key, int pos, Object value) {
            int[] table = keys;
            int maxindex = table.length;
            int index = (key & 0x7fffffff) % maxindex;

            // Fairly aribtrary choice for stepsize...
            int stepsize = maxindex / 5;

            // Cycle through possible positions for the key;
            while (true) {
                int tkey = table[index];
                if (values[index] == null) {
                    table[index] = key;
                    position[index] = pos;
                    values[index] = value;
                    filled++;
                    size++;
                    break;
                } else if (tkey == key && values[index] == value) {
                    position[index] = pos;
                    break;
                }
                index = (index+stepsize) % maxindex;
            }
        }


        private final void resize(int capacity) {
            int p = prime;
            for(; p<primes.length; p++) {
                if (primes[p] >= capacity) break;
            }
            if (primes[p] < capacity) {
                throw Py.ValueError("can't make hashtable of size: " +
                                    capacity);
            }
            capacity = primes[p];
            prime = p;

            int[] oldKeys = keys;
            int[] oldPositions = position;
            Object[] oldValues = values;

            keys = new int[capacity];
            position = new int[capacity];
            values = new Object[capacity];
            size = 0;
            filled = 0;

            if (oldValues != null) {
                int n = oldValues.length;

                for(int i=0; i<n; i++) {
                    Object value = oldValues[i];
                    if (value == null) continue;
                    insertkey(oldKeys[i], oldPositions[i], value);
                }
            }
        }

        public void put(int key, int pos, Object value) {
            if (2*filled > keys.length) resize(keys.length+1);
            insertkey(key, pos, value);
        }
    }






    /**
     * The Unpickler object. Unpickler instances are create by the factory
     * methods Unpickler.
     * @see cPickle#Unpickler(PyObject)
     */
    static public class Unpickler {

        private PyIOFile file;

        public Map<String,PyObject> memo = Generic.map();

        /**
         * For the benefit of persistency modules written using pickle,
         * it supports the notion of a reference to an object outside
         * the pickled data stream.
         * Such objects are referenced by a name, which is an arbitrary
         * string of printable ASCII characters.
         * The resolution of such names is not defined by the pickle module
         * -- the persistent object module will have to add a method
         * persistent_load().
         */
        public PyObject persistent_load = null;
        public PyObject find_global = null;

        private PyObject mark = new PyString("spam");

        private int stackTop;
        private PyObject[] stack;


        Unpickler(PyObject file) {
            this.file = PyIOFileFactory.createIOFile(file);
        }


        /**
         * Unpickle and return an instance of the object represented by
         * the file.
         */
        public PyObject load() {
            stackTop = 0;
            stack = new PyObject[10];

            while (true) {
                String s = file.read(1);
//              System.out.println("load:" + s);
//              for (int i = 0; i < stackTop; i++)
//                  System.out.println("   " + stack[i]);
                if (s.length() < 1)
                    load_eof();
                char key = s.charAt(0);
                switch (key) {
                case PERSID:          load_persid(); break;
                case BINPERSID:       load_binpersid(); break;
                case NONE:            load_none(); break;
                case INT:             load_int(); break;
                case BININT:          load_binint(); break;
                case BININT1:         load_binint1(); break;
                case BININT2:         load_binint2(); break;
                case LONG:            load_long(); break;
                case FLOAT:           load_float(); break;
                case BINFLOAT:        load_binfloat(); break;
                case STRING:          load_string(); break;
                case BINSTRING:       load_binstring(); break;
                case SHORT_BINSTRING: load_short_binstring(); break;
                case UNICODE:         load_unicode(); break;
                case BINUNICODE:      load_binunicode(); break;
                case TUPLE:           load_tuple(); break;
                case EMPTY_TUPLE:     load_empty_tuple(); break;
                case EMPTY_LIST:      load_empty_list(); break;
                case EMPTY_DICT:      load_empty_dictionary(); break;
                case LIST:            load_list(); break;
                case DICT:            load_dict(); break;
                case INST:            load_inst(); break;
                case OBJ:             load_obj(); break;
                case GLOBAL:          load_global(); break;
                case REDUCE:          load_reduce(); break;
                case POP:             load_pop(); break;
                case POP_MARK:        load_pop_mark(); break;
                case DUP:             load_dup(); break;
                case GET:             load_get(); break;
                case BINGET:          load_binget(); break;
                case LONG_BINGET:     load_long_binget(); break;
                case PUT:             load_put(); break;
                case BINPUT:          load_binput(); break;
                case LONG_BINPUT:     load_long_binput(); break;
                case APPEND:          load_append(); break;
                case APPENDS:         load_appends(); break;
                case SETITEM:         load_setitem(); break;
                case SETITEMS:        load_setitems(); break;
                case BUILD:           load_build(); break;
                case MARK:            load_mark(); break;
                case PROTO:           load_proto(); break;
                case NEWOBJ:          load_newobj(); break;
                case EXT1:            load_ext(1); break;
                case EXT2:            load_ext(2); break;
                case EXT4:            load_ext(4); break;
                case TUPLE1:          load_small_tuple(1); break;
                case TUPLE2:          load_small_tuple(2); break;
                case TUPLE3:          load_small_tuple(3); break;
                case NEWTRUE:         load_boolean(true); break;
                case NEWFALSE:        load_boolean(false); break;
                case LONG1:           load_bin_long(1); break;
                case LONG4:           load_bin_long(4); break;
                case STOP:
                    return load_stop();
                default:
                    throw new PyException(UnpicklingError,
                                          String.format("invalid load key, '%s'.", key));
                }
            }
        }


        final private int marker() {
            for (int k = stackTop-1; k >= 0; k--)
                if (stack[k] == mark)
                    return stackTop-k-1;
            throw new PyException(UnpicklingError,
                        "Inputstream corrupt, marker not found");
        }


        final private void load_eof() {
            throw new PyException(Py.EOFError);
        }

        private void load_proto() {
            int proto = file.read(1).charAt(0);
            if (proto < 0 || proto > 2)
                throw Py.ValueError("unsupported pickle protocol: " + proto);
        }


        final private void load_persid() {
            load_persid(new PyString(file.readlineNoNl()));
        }


        final private void load_binpersid() {
            load_persid(pop());
        }

        final private void load_persid(PyObject pid) {
            if (persistent_load == null) {
                throw new PyException(UnpicklingError,
                                      "A load persistent id instruction was encountered,\n"
                                      + "but no persistent_load function was specified.");
            }

            if (persistent_load instanceof PyList) {
                ((PyList)persistent_load).append(pid);
            } else {
                pid = persistent_load.__call__(pid);
            }
            push(pid);
        }


        final private void load_none() {
            push(Py.None);
        }

        final private void load_int() {
            String line = file.readlineNoNl();
            PyObject value;
            // The following could be abstracted into a common string
            // -> int/long method.
            if (line.equals("01")) {
                value = Py.True;
            }
            else if (line.equals("00")) {
                value = Py.False;
            }
            else {
                try {
                    value = Py.newInteger(Integer.parseInt(line));
                } catch(NumberFormatException e) {
                    try {
                        value = Py.newLong(line);
                    } catch(NumberFormatException e2) {
                        throw Py.ValueError("could not convert string to int");
                    }
                }
            }
            push(value);
        }

        private void load_boolean(boolean value) {
            push(value ? Py.True : Py.False);
        }

        final private void load_binint() {
            int x = read_binint();
            push(new PyInteger(x));
        }

        private int read_binint() {
            String s = file.read(4);
            return s.charAt(0) |
                   (s.charAt(1)<<8) |
                   (s.charAt(2)<<16) |
                   (s.charAt(3)<<24);
        }


        final private void load_binint1() {
            int val = file.read(1).charAt(0);
            push(new PyInteger(val));
        }

        final private void load_binint2() {
            int val = read_binint2();
            push(new PyInteger(val));
        }

        private int read_binint2() {
            String s = file.read(2);
            return (s.charAt(1)) << 8 | (s.charAt(0));
        }


        final private void load_long() {
            String line = file.readlineNoNl();
            push(new PyLong(line.substring(0, line.length()-1)));
        }

        private void load_bin_long(int length) {
            int longLength = read_binint(length);
            if (longLength == 0) {
                push(new PyLong(BigInteger.ZERO));
                return;
            }
            String s = file.read(longLength);
            byte[] bytes = new byte[s.length()];
            // Write to the byte array in reverse order: pickle orders
            // by little endian whereas BigInteger orders by big
            // endian
            int n = s.length() - 1;
            for (int i = 0; i < s.length(); i++, n--) {
                char c = s.charAt(i);
                if(c >= 128) {
                    bytes[n] = (byte)(c - 256);
                } else {
                    bytes[n] = (byte)c;
                }
            }
            BigInteger bigint = new BigInteger(bytes);
            push(new PyLong(bigint));
        }

        private int read_binint(int length) {
            if (length == 1)
                return file.read(1).charAt(0);
            else if (length == 2)
                return read_binint2();
            else
                return read_binint();
        }

        final private void load_float() {
            String line = file.readlineNoNl();
            push(new PyFloat(Double.valueOf(line).doubleValue()));
        }

        final private void load_binfloat() {
            String s = file.read(8);
            long bits = s.charAt(7) |
                        ((long)s.charAt(6) << 8) |
                        ((long)s.charAt(5) << 16) |
                        ((long)s.charAt(4) << 24) |
                        ((long)s.charAt(3) << 32) |
                        ((long)s.charAt(2) << 40) |
                        ((long)s.charAt(1) << 48) |
                        ((long)s.charAt(0) << 56);
            push(new PyFloat(Double.longBitsToDouble(bits)));
        }

        final private void load_string() {
            String line = file.readlineNoNl();

            String value;
            char quote = line.charAt(0);
            if (quote != '"' && quote != '\'')
                throw Py.ValueError("insecure string pickle");

            int nslash = 0;
            int i;
            char ch = '\0';
            int n = line.length();
            for (i = 1; i < n; i++) {
                ch = line.charAt(i);
                if (ch == quote && nslash % 2 == 0)
                    break;
                if (ch == '\\')
                    nslash++;
                else
                    nslash = 0;
            }
            if (ch != quote)
                throw Py.ValueError("insecure string pickle");

            for (i++ ; i < line.length(); i++) {
                if (line.charAt(i) > ' ')
                    throw Py.ValueError("insecure string pickle " + i);
            }
            value = PyString.decode_UnicodeEscape(line, 1, n-1,
                                                  "strict", false);

            push(new PyString(value));
        }


        final private void load_binstring() {
            int len = read_binint();
            push(new PyString(file.read(len)));
        }


        final private void load_short_binstring() {
            int len = file.read(1).charAt(0);
            push(new PyString(file.read(len)));
        }


        final private void load_unicode() {
            String line = file.readlineNoNl();
            String value = codecs.PyUnicode_DecodeRawUnicodeEscape(line,
                                                                   "strict");
            push(new PyUnicode(value));
        }

        final private void load_binunicode() {
            int len = read_binint();
            String line = file.read(len);
            push(new PyUnicode(codecs.PyUnicode_DecodeUTF8(line, "strict")));
        }

        final private void load_tuple() {
            PyObject[] arr = new PyObject[marker()];
            pop(arr);
            pop();
            push(new PyTuple(arr));
        }

        final private void load_empty_tuple() {
            push(new PyTuple(Py.EmptyObjects));
        }

        private void load_small_tuple(int length) {
            PyObject[] data = new PyObject[length];
            for(int i=length-1; i >= 0; i--) {
                data [i] = pop();
            }
            push(new PyTuple(data));
        }

        final private void load_empty_list() {
            push(new PyList(Py.EmptyObjects));
        }

        final private void load_empty_dictionary() {
            push(new PyDictionary());
        }


        final private void load_list() {
            PyObject[] arr = new PyObject[marker()];
            pop(arr);
            pop();
            push(new PyList(arr));
        }


        final private void load_dict() {
            int k = marker();
            PyDictionary d = new PyDictionary();
            for (int i = 0; i < k; i += 2) {
                PyObject value = pop();
                PyObject key = pop();
                d.__setitem__(key, value);
            }
            pop();
            push(d);
        }


        final private void load_inst() {
            PyObject[] args = new PyObject[marker()];
            pop(args);
            pop();

            String module = file.readlineNoNl();
            String name = file.readlineNoNl();
            PyObject klass = find_class(module, name);

            PyObject value = null;
            if (args.length == 0 && klass instanceof PyClass &&
                        klass.__findattr__("__getinitargs__") == null) {
                value = new PyInstance((PyClass)klass);
            } else {
                value = klass.__call__(args);
            }
            push(value);
        }


        final private void load_obj() {
            PyObject[] args = new PyObject[marker()-1];
            pop(args);
            PyObject klass = pop();
            pop();

            PyObject value = null;
            if (args.length == 0 && klass instanceof PyClass &&
                        klass.__findattr__("__getinitargs__") == null) {
                value = new PyInstance((PyClass)klass);
            } else {
                value = klass.__call__(args);
            }
            push(value);
        }

        final private void load_global() {
            String module = file.readlineNoNl();
            String name = file.readlineNoNl();
            PyObject klass = find_class(module, name);
            push(klass);
        }


        final private PyObject find_class(String module, String name) {
            if (find_global != null) {
               if (find_global == Py.None)
                   throw new PyException(UnpicklingError,
                         "Global and instance pickles are not supported.");
               return find_global.__call__(new PyString(module), new PyString(name));
            }

            PyObject modules = Py.getSystemState().modules;
            PyObject mod = modules.__finditem__(module.intern());
            if (mod == null) {
                mod = importModule(module);
            }
            PyObject global = mod.__findattr__(name.intern());
            if (global == null) {
                throw new PyException(Py.SystemError,
                          "Failed to import class " + name + " from module " +
                          module);
            }
            return global;
        }

        private void load_ext(int length) {
            int code = read_binint(length);
            // TODO: support _extension_cache
            PyObject key = inverted_registry.get(Py.newInteger(code));
            if (key == null) {
                throw new PyException(Py.ValueError, "unregistered extension code " + code);
            }
            String module = key.__finditem__(0).toString();
            String name = key.__finditem__(1).toString();
            push(find_class(module, name));
        }


        final private void load_reduce() {
            PyObject arg_tup = pop();
            PyObject callable = pop();
            PyObject value = null;
            if (arg_tup == Py.None) {
                // XXX __basicnew__ ?
                value = callable.__findattr__("__basicnew__").__call__();
            } else {
                value = callable.__call__(make_array(arg_tup));
            }
            push(value);
        }

        private void load_newobj() {
            PyObject arg_tup = pop();
            PyObject cls = pop();
            PyObject[] args = new PyObject[arg_tup.__len__() + 1];
            args [0] = cls;
            for(int i=1; i<args.length; i++) {
                args [i] = arg_tup.__finditem__(i-1);
            }
            push(cls.__getattr__("__new__").__call__(args));
        }

        final private PyObject[] make_array(PyObject seq) {
            int n = seq.__len__();
            PyObject[] objs= new PyObject[n];

            for(int i=0; i<n; i++)
                objs[i] = seq.__finditem__(i);
            return objs;
        }

        final private void load_pop() {
            pop();
        }


        final private void load_pop_mark() {
            pop(marker());
        }

        final private void load_dup() {
            push(peek());
        }

        final private void load_get() {
            String py_str = file.readlineNoNl();
            PyObject value = memo.get(py_str);
            if (value == null) {
                throw new PyException(BadPickleGet, py_str);
            }
            push(value);
        }

        final private void load_binget() {
            String py_key = String.valueOf((int)file.read(1).charAt(0));
            PyObject value = memo.get(py_key);
            if (value == null) {
                throw new PyException(BadPickleGet, py_key);
            }
            push(value);
        }

        final private void load_long_binget() {
            int i = read_binint();
            String py_key = String.valueOf(i);
            PyObject value = memo.get(py_key);
            if (value == null) {
                throw new PyException(BadPickleGet, py_key);
            }
            push(value);
        }


        final private void load_put() {
            memo.put(file.readlineNoNl(), peek());
        }


        final private void load_binput() {
            int i = file.read(1).charAt(0);
            memo.put(String.valueOf(i), peek());
        }


        final private void load_long_binput() {
            int i = read_binint();
            memo.put(String.valueOf(i), peek());
        }

        final private void load_append() {
            PyObject value = pop();
            PyObject obj = peek();
            if(obj instanceof PyList) {
                ((PyList)obj).append(value);
            } else {
                PyObject appender = obj.__getattr__("append");
                appender.__call__(value);
            }
        }

        final private void load_appends() {
            int mark = marker();
            PyObject obj = peek(mark + 1);
            if(obj instanceof PyList) {
                for(int i = mark - 1; i >= 0; i--) {
                    ((PyList)obj).append(peek(i));
                }
            } else {
                PyObject appender = obj.__getattr__("append");
                for(int i = mark - 1; i >= 0; i--) {
                    appender.__call__(peek(i));
                }
            }
            pop(mark + 1);
        }

        final private void load_setitem() {
            PyObject value = pop();
            PyObject key   = pop();
            PyDictionary dict = (PyDictionary)peek();
            dict.__setitem__(key, value);
        }


        final private void load_setitems() {
            int mark = marker();
            PyDictionary dict = (PyDictionary)peek(mark+1);
            for (int i = 0; i < mark; i += 2) {
                PyObject key   = peek(i+1);
                PyObject value = peek(i);
                dict.__setitem__(key, value);
            }
            pop(mark+1);
        }

        private void load_build() {
            PyObject state = pop();
            PyObject inst  = peek();
            PyObject setstate = inst.__findattr__("__setstate__");
            if (setstate != null) {
                // The explicit __setstate__ is responsible for everything.
                setstate.__call__(state);
                return;
            }

            // A default __setstate__.  First see whether state embeds a slot state dict
            // too (a proto 2 addition).
            PyObject slotstate = null;
            if (state instanceof PyTuple && state.__len__() == 2) {
                PyObject temp = state;
                state = temp.__getitem__(0);
                slotstate = temp.__getitem__(1);
            }

            if (state != Py.None) {
                if (!(state instanceof PyDictionary)) {
                    throw new PyException(UnpicklingError, "state is not a dictionary");
                }
                PyObject dict = inst.__getattr__("__dict__");
                for (PyObject item : ((PyDictionary)state).iteritems().asIterable()) {
                    dict.__setitem__(item.__getitem__(0), item.__getitem__(1));
                }
            }

            // Also set instance attributes from the slotstate dict (if any).
            if (slotstate != null) {
                if (!(slotstate instanceof PyDictionary)) {
                    throw new PyException(UnpicklingError, "slot state is not a dictionary");
                }
                for (PyObject item : ((PyDictionary)slotstate).iteritems().asIterable()) {
                    inst.__setattr__(PyObject.asName(item.__getitem__(0)),
                                     item.__getitem__(1));
                }
            }
        }

        final private void load_mark() {
            push(mark);
        }

        final private PyObject load_stop() {
            return pop();
        }



        final private PyObject peek() {
            return stack[stackTop-1];
        }

        final private PyObject peek(int count) {
            return stack[stackTop-count-1];
        }


        final private PyObject pop() {
            PyObject val = stack[--stackTop];
            stack[stackTop] = null;
            return val;
        }

        final private void pop(int count) {
            for (int i = 0; i < count; i++)
                stack[--stackTop] = null;
        }


        final private void pop(PyObject[] arr) {
            int len = arr.length;
            System.arraycopy(stack, stackTop - len, arr, 0, len);
            stackTop -= len;
        }

        final private void push(PyObject val) {
            if (stackTop >= stack.length) {
                PyObject[] newStack = new PyObject[(stackTop+1) * 2];
                System.arraycopy(stack, 0, newStack, 0, stack.length);
                stack = newStack;
            }
            stack[stackTop++] = val;
        }
    }


    private static PyObject importModule(String name) {
        PyObject fromlist = new PyTuple(Py.newString("__doc__"));
        return __builtin__.__import__(name, Py.None, Py.None, fromlist);
    }

    private static PyObject getJavaFunc(String name, String methodName) {
        return exceptions.bindStaticJavaMethod(name, cPickle.class, methodName);
    }
}
