/*
 * JBoss, Home of Professional Open Source
 *
 * Distributable under LGPL license.
 * See terms of license at gnu.org.
 */
package org.jboss.cache.invalidation;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jboss.cache.CacheSPI;
import org.jboss.cache.DefaultCacheFactory;
import org.jboss.cache.Fqn;
import org.jboss.cache.Node;
import org.jboss.cache.NodeSPI;
import org.jboss.cache.util.TestingUtil;
import org.jboss.cache.util.internals.ReplicationListener;
import org.jboss.cache.config.CacheLoaderConfig;
import org.jboss.cache.config.Configuration;
import org.jboss.cache.factories.XmlConfigurationParser;
import org.jboss.cache.optimistic.DefaultDataVersion;
import org.jboss.cache.xml.XmlHelper;
import static org.testng.AssertJUnit.*;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
import org.w3c.dom.Element;

import javax.transaction.RollbackException;
import javax.transaction.Transaction;
import javax.transaction.TransactionManager;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * Tests the async interceptor
 *
 * @author <a href="mailto:manik@jboss.org">Manik Surtani (manik@jboss.org)</a>
 */
@Test(groups = {"functional", "jgroups"})
public class InvalidationInterceptorTest
{
   private static Log log = LogFactory.getLog(InvalidationInterceptorTest.class);
   private CacheSPI<Object, Object> cache1, cache2;
   private Set<CacheSPI> toClean = new HashSet<CacheSPI>();

   @AfterMethod
   public void tearDown()
   {
      TestingUtil.killCaches(cache1, cache2);
      for (CacheSPI c : toClean) TestingUtil.killCaches(c);
      toClean.clear();
   }


   public void testPessimisticNonTransactional() throws Exception
   {
      cache1 = createCache(false);
      cache2 = createCache(false);

      Fqn fqn = Fqn.fromString("/a/b");
      cache1.put(fqn, "key", "value");

      // test that this has NOT replicated, but rather has been invalidated:
      assertEquals("value", cache1.get(fqn, "key"));
      assertNull("Should NOT have replicated!", cache2.getNode(fqn));

      log.info("***** Node not replicated, as expected.");

      // now make sure cache2 is in sync with cache1:
      cache2.put(fqn, "key", "value");

      // since the node already exists even PL will not remove it - but will invalidate it's data
      Node n = cache1.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
      assertEquals("value", cache2.get(fqn, "key"));

      // now test the invalidation:
      cache1.put(fqn, "key2", "value2");
      assertEquals("value2", cache1.get(fqn, "key2"));
      n = cache2.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
   }

   public void testUnnecessaryEvictions() throws Exception
   {
      cache1 = createCache(false);
      cache2 = createCache(false);

      Fqn fqn1 = Fqn.fromString("/a/b/c");
      Fqn fqn2 = Fqn.fromString("/a/b/d");

      cache1.put(fqn1, "hello", "world");

      assertEquals("world", cache1.get(fqn1, "hello"));
      assertNull(cache2.get(fqn1, "hello"));

      cache2.put(fqn2, "hello", "world");
      assertEquals("world", cache1.get(fqn1, "hello"));
      assertNull(cache2.get(fqn1, "hello"));
      assertEquals("world", cache2.get(fqn2, "hello"));
      assertNull(cache1.get(fqn2, "hello"));

      cache2.put(fqn1, "hello", "world");
      assertEquals("world", cache2.get(fqn1, "hello"));
      assertEquals("world", cache2.get(fqn2, "hello"));
      assertNull(cache1.get(fqn1, "hello"));
      assertNull(cache1.get(fqn2, "hello"));
   }


   public void testPessimisticNonTransactionalAsync() throws Exception
   {
      cache1 = createUnstartedCache(false);
      cache2 = createUnstartedCache(false);
      cache1.getConfiguration().setCacheMode(Configuration.CacheMode.INVALIDATION_ASYNC);
      cache2.getConfiguration().setCacheMode(Configuration.CacheMode.INVALIDATION_ASYNC);
      cache1.start();
      cache2.start();

      Fqn fqn = Fqn.fromString("/a/b");
      ReplicationListener replListener1 = new ReplicationListener(cache1);
      ReplicationListener replListener2 = new ReplicationListener(cache2);

      replListener2.expectAny();
      cache1.put(fqn, "key", "value");
      replListener2.waitForReplicationToOccur(500);
//      TestingUtil.sleepThread(500);// give it time to broadcast the evict call
      // test that this has NOT replicated, but rather has been invalidated:
      assertEquals("value", cache1.get(fqn, "key"));
      assertNull("Should NOT have replicated!", cache2.getNode(fqn));

      replListener1.expectAny();
      // now make sure cache2 is in sync with cache1:
      cache2.put(fqn, "key", "value");
//      TestingUtil.sleepThread(500);// give it time to broadcast the evict call
      replListener1.waitForReplicationToOccur(500);

      // since the node already exists even PL will not remove it - but will invalidate it's data
      Node n = cache1.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
      assertEquals("value", cache2.get(fqn, "key"));

      replListener2.expectAny();
      // now test the invalidation:
      cache1.put(fqn, "key2", "value2");
      assertEquals("value2", cache1.get(fqn, "key2"));
//      TestingUtil.sleepThread(500);// give it time to broadcast the evict call
      replListener2.waitForReplicationToOccur(500);

      // since the node already exists even PL will not remove it - but will invalidate it's data
      n = cache2.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
   }


   public void testPessimisticTransactional() throws Exception
   {
      cache1 = createCache(false);
      cache2 = createCache(false);

      Fqn fqn = Fqn.fromString("/a/b");
      cache1.put(fqn, "key", "value");

      // test that this has NOT replicated, but rather has been invalidated:
      assertEquals("value", cache1.get(fqn, "key"));
      assertNull("Should NOT have replicated!", cache2.getNode(fqn));

      log.info("***** Node not replicated, as expected.");

      // now make sure cache2 is in sync with cache1:
      // make sure this is in a tx
      TransactionManager txm = cache2.getTransactionManager();
      assertEquals("value", cache1.get(fqn, "key"));

      txm.begin();
      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));
      txm.commit();

      // since the node already exists even PL will not remove it - but will invalidate it's data
      Node n = cache1.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
      assertEquals("value", cache2.get(fqn, "key"));

      // now test the invalidation again
      txm = cache1.getTransactionManager();
      assertEquals("value", cache2.get(fqn, "key"));

      txm.begin();
      cache1.put(fqn, "key2", "value2");
      assertEquals("value2", cache1.get(fqn, "key2"));
      txm.commit();

      assertEquals("value2", cache1.get(fqn, "key2"));
      // since the node already exists even PL will not remove it - but will invalidate it's data
      n = cache2.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");

      // test a rollback
      txm = cache2.getTransactionManager();
      assertEquals("value2", cache1.get(fqn, "key2"));

      txm.begin();
      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));
      txm.rollback();

      assertEquals("value2", cache1.get(fqn, "key2"));
      n = cache2.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
   }


   public void testOptSyncUnableToEvict() throws Exception
   {
      cache1 = createCache(true);
      cache2 = createCache(true);

      Fqn fqn = Fqn.fromString("/a/b");

      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));
      Node n = cache1.getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
      assertHasBeenInvalidated(cache1.peek(fqn, true, true), "Should have been invalidated");

      // start a tx that cache1 will have to send out an evict ...
      TransactionManager mgr1 = cache1.getTransactionManager();
      TransactionManager mgr2 = cache2.getTransactionManager();

      mgr1.begin();
      cache1.put(fqn, "key2", "value2");
      Transaction tx1 = mgr1.suspend();
      mgr2.begin();
      cache2.put(fqn, "key3", "value3");
      Transaction tx2 = mgr2.suspend();
      mgr1.resume(tx1);
      // this oughtta fail
      try
      {
         mgr1.commit();
         assertTrue("Ought to have succeeded!", true);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have succeeded!", false);
      }

      mgr2.resume(tx2);
      try
      {
         mgr2.commit();
         assertTrue("Ought to have failed!", false);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have failed!", true);
      }
   }

   public void testPessTxSyncUnableToEvict() throws Exception
   {
      cache1 = createCache(false);
      cache2 = createCache(false);

      Fqn fqn = Fqn.fromString("/a/b");

      cache1.put("/a/b", "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      assertNull(cache2.getNode(fqn));

      // start a tx that cacahe1 will have to send out an evict ...
      TransactionManager mgr1 = cache1.getTransactionManager();
      TransactionManager mgr2 = cache2.getTransactionManager();

      mgr1.begin();
      cache1.put(fqn, "key2", "value2");
      Transaction tx1 = mgr1.suspend();
      mgr2.begin();
      cache2.put(fqn, "key3", "value3");
      Transaction tx2 = mgr2.suspend();
      mgr1.resume(tx1);
      // this oughtta fail
      try
      {
         mgr1.commit();
         assertTrue("Ought to have failed!", false);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have failed!", true);
      }

      mgr2.resume(tx2);
      try
      {
         mgr2.commit();
         assertTrue("Ought to have succeeded!", true);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have succeeded!", false);
      }
   }

   public void testPessTxAsyncUnableToEvict() throws Exception
   {
      cache1 = createUnstartedCache(false);
      cache2 = createUnstartedCache(false);
      cache1.getConfiguration().setCacheMode(Configuration.CacheMode.INVALIDATION_ASYNC);
      cache2.getConfiguration().setCacheMode(Configuration.CacheMode.INVALIDATION_ASYNC);
      cache1.start();
      cache2.start();

      Fqn fqn = Fqn.fromString("/a/b");

      cache1.put("/a/b", "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      assertNull(cache2.getNode(fqn));

      // start a tx that cacahe1 will have to send out an evict ...
      TransactionManager mgr1 = cache1.getTransactionManager();
      TransactionManager mgr2 = cache2.getTransactionManager();

      mgr1.begin();
      cache1.put(fqn, "key2", "value2");
      Transaction tx1 = mgr1.suspend();
      mgr2.begin();
      cache2.put(fqn, "key3", "value3");
      Transaction tx2 = mgr2.suspend();
      mgr1.resume(tx1);
      // this oughtta fail
      try
      {
         mgr1.commit();
         assertTrue("Ought to have succeeded!", true);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have succeeded!", false);
      }

      mgr2.resume(tx2);
      try
      {
         mgr2.commit();
         assertTrue("Ought to have succeeded!", true);
      }
      catch (RollbackException roll)
      {
         assertTrue("Ought to have succeeded!", false);
      }
   }

   public void testPessimisticNodeRemoval() throws Exception
   {
      nodeRemovalTest(false);
   }

   public void testOptimisticNodeRemoval() throws Exception
   {
      nodeRemovalTest(true);
   }

   @SuppressWarnings("unchecked")
   private void nodeRemovalTest(boolean optimistic) throws Exception
   {
      cache1 = createCache(optimistic);
      cache2 = createCache(optimistic);

      Node root1 = cache1.getRoot();
      Node root2 = cache2.getRoot();

      // this fqn is relative, but since it is from the root it may as well be absolute
      Fqn<String> fqn = Fqn.fromString("/test/fqn");
      cache1.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      cache1.put(fqn, "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      cache2.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));

      assertEquals(true, cache1.removeNode(fqn));
      assertFalse(root1.hasChild(fqn));
      Node remoteNode = root2.getChild(fqn);
      checkRemoteNodeIsRemoved(remoteNode);
      assertEquals(false, cache1.removeNode(fqn));

      Fqn<String> child = Fqn.fromString("/test/fqn/child");
      cache1.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      cache1.put(child, "key", "value");
      assertEquals("value", cache1.get(child, "key"));
      cache2.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      cache2.put(child, "key", "value");
      assertEquals("value", cache2.get(child, "key"));

      assertEquals(true, cache1.removeNode(fqn));
      assertFalse(root1.hasChild(fqn));
      remoteNode = root2.getChild(fqn);
      checkRemoteNodeIsRemoved(remoteNode);
      assertEquals(false, cache1.removeNode(fqn));
   }

   private void checkRemoteNodeIsRemoved(Node<Object, Object> remoteNode)
   {
      assertHasBeenInvalidated(remoteNode, "Should have been removed");
      // Recursively check any children
      if (remoteNode != null)
      {
         for (Node<Object, Object> child : remoteNode.getChildren())
         {
            checkRemoteNodeIsRemoved(child);
         }
      }
   }

   public void testPessimisticNodeResurrection() throws Exception
   {
      nodeResurrectionTest(false);
   }

   public void testOptimisticNodeResurrection() throws Exception
   {
      nodeResurrectionTest(true);
   }

   private void nodeResurrectionTest(boolean optimistic) throws Exception
   {
      cache1 = createCache(optimistic);
      cache2 = createCache(optimistic);

      // this fqn is relative, but since it is from the root it may as well be absolute
      Fqn<String> fqn = Fqn.fromString("/test/fqn1");
      cache1.put(fqn, "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));
      // Change the value in order to increment the version if Optimistic is used
      cache1.put(fqn, "key", "newValue");
      assertEquals("newValue", cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));

      assertEquals(true, cache1.removeNode(fqn));
      assertEquals(null, cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));

      // Restore locally
      cache1.put(fqn, "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));

      // Repeat, but now restore the node on the remote cache
      fqn = Fqn.fromString("/test/fqn2");
      cache1.put(fqn, "key", "value");
      assertEquals("value", cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));
      // Change the value in order to increment the version if Optimistic is used
      cache1.put(fqn, "key", "newValue");
      assertEquals("newValue", cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));

      assertEquals(true, cache1.removeNode(fqn));
      assertEquals(null, cache1.get(fqn, "key"));
      assertEquals(null, cache2.get(fqn, "key"));

      // Restore on remote cache
      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));
      assertEquals(null, cache1.get(fqn, "key"));
   }

   /**
    * Test for JBCACHE-1251.
    *
    * @throws Exception
    */
   public void testPessimisticNodeResurrection2() throws Exception
   {
      nodeResurrectionTest2(false);
   }

   /**
    * OPTIMISTIC locking verion of test for JBCACHE-1251. JBCACHE-1251
    * did not effect optimistic, but we add the test to guard against
    * regressions.
    *
    * @throws Exception
    */
   public void testOptimisticNodeResurrection2() throws Exception
   {
      nodeResurrectionTest2(true);
   }

   /**
    * Here we model a scenario where a parent node represents
    * a structural node, and then child nodes represent different
    * data elements.
    * <p/>
    * Such data structures are set up on both caches, and then the parent node
    * is removed (globally) and re-added (locally) on one cache.  This
    * represents an attempt to clear the region -- removing a node and
    * re-adding is one of the only ways to do this.
    * <p/>
    * On the second cache, the fact that the structural node is missing is
    * detected, and an attempt is made to re-add it locally.
    *
    * @param optimistic should the cache be configured for optimistic locking
    * @throws Exception
    */
   private void nodeResurrectionTest2(boolean optimistic) throws Exception
   {
      cache1 = createCache(optimistic);
      cache2 = createCache(optimistic);

      Node root1 = cache1.getRoot();
      Node root2 = cache2.getRoot();

      // this fqn is relative, but since it is from the root it may as well be absolute
      Fqn<String> fqn = Fqn.fromString("/test/fqn");
      cache1.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      root1.addChild(fqn);
      assertEquals(true, root1.hasChild(fqn));
      cache2.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      root1.addChild(fqn);
      assertEquals(true, root1.hasChild(fqn));

      Fqn child = Fqn.fromRelativeElements(fqn, "child");
      cache1.putForExternalRead(child, "key", "value");
      cache2.putForExternalRead(child, "key", "value");
      assertEquals("value", cache1.get(child, "key"));
      assertEquals("value", cache2.get(child, "key"));

      assertEquals(true, cache1.removeNode(fqn));
      assertFalse(root1.hasChild(fqn));

      cache1.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      root1.addChild(fqn);
      assertEquals(true, root1.hasChild(fqn));

      Node remoteNode = root2.getChild(fqn);
      checkRemoteNodeIsRemoved(remoteNode);
      cache2.getInvocationContext().getOptionOverrides().setCacheModeLocal(true);
      root2.addChild(fqn);
      assertEquals(true, root2.hasChild(fqn));
   }

   private void dumpVersionInfo(CacheSPI c1, CacheSPI c2, Fqn fqn)
   {
      System.out.println("**** Versin Info for Fqn [" + fqn + "] ****");
      NodeSPI n1 = c1.getRoot().getChildDirect(fqn);
      System.out.println("  Cache 1: " + n1.getVersion() + " dataLoaded? " + n1.isDataLoaded());

      NodeSPI n2 = c2.getRoot().getChildDirect(fqn);
      System.out.println("  Cache 2: " + n2.getVersion() + " dataLoaded? " + n2.isDataLoaded());
   }


   public void testOptimistic() throws Exception
   {
      cache1 = createCache(true);
      cache2 = createCache(true);

      Fqn fqn = Fqn.fromString("/a/b");
      cache1.put(fqn, "key", "value");

      dumpVersionInfo(cache1, cache2, fqn);

      // test that this has NOT replicated, but rather has been invalidated:
      assertEquals("value", cache1.get(fqn, "key"));
      Node n2 = cache2.getNode(fqn);
      assertHasBeenInvalidated(n2, "Should have been invalidated");
      assertHasBeenInvalidated(cache2.peek(fqn, true, true), "Should have been invalidated");

      // now make sure cache2 is in sync with cache1:
      cache2.put(fqn, "key", "value");

      dumpVersionInfo(cache1, cache2, fqn);

      Node n1 = cache1.getNode(fqn);
      assertHasBeenInvalidated(n1, "Should have been invalidated");
      assertHasBeenInvalidated(cache1.peek(fqn, true, true), "Should have been invalidated");

      assertEquals("value", cache2.get(fqn, "key"));

      // now test the invalidation:
      cache1.put(fqn, "key2", "value2");

      dumpVersionInfo(cache1, cache2, fqn);

      assertEquals("value2", cache1.get(fqn, "key2"));
      n2 = cache2.getNode(fqn);
      assertHasBeenInvalidated(n2, "Should have been invalidated");
      assertHasBeenInvalidated(cache2.peek(fqn, false, false), "Should have been invalidated");

      // with tx's
      TransactionManager txm = cache2.getTransactionManager();

      txm.begin();
      cache2.put(fqn, "key", "value");
      assertEquals("value", cache2.get(fqn, "key"));
      assertEquals("value2", cache1.get(fqn, "key2"));
      txm.commit();

      n1 = cache1.getNode(fqn);
      assertHasBeenInvalidated(n1, "Should have been invalidated");
      assertHasBeenInvalidated(cache1.peek(fqn, false, false), "Should have been invalidated");
      assertEquals("value", cache2.get(fqn, "key"));

      // now test the invalidation again
      txm = cache1.getTransactionManager();

      txm.begin();
      cache1.put(fqn, "key2", "value2");
      assertEquals("value", cache2.get(fqn, "key"));
      assertEquals("value2", cache1.get(fqn, "key2"));
      txm.commit();

      assertEquals("value2", cache1.get(fqn, "key2"));
      n2 = cache2.getNode(fqn);
      assertHasBeenInvalidated(n2, "Should have been invalidated");
      assertHasBeenInvalidated(cache2.peek(fqn, false, false), "Should have been invalidated");

      // test a rollback
      txm = cache2.getTransactionManager();

      txm.begin();
      cache2.put(fqn, "key", "value");
      assertEquals("value2", cache1.get(fqn, "key2"));
      assertEquals("value", cache2.get(fqn, "key"));
      txm.rollback();

      assertEquals("value2", cache1.get(fqn, "key2"));
      n2 = cache2.getNode(fqn);
      assertHasBeenInvalidated(n2, "Should have been invalidated");
      assertHasBeenInvalidated(cache2.peek(fqn, false, false), "Should have been invalidated");
   }

   public void testPessimisticNonTransactionalWithCacheLoader() throws Exception
   {
      List<CacheSPI<Object, Object>> caches = createCachesWithSharedCL(false);
      cache1 = caches.get(0);
      cache2 = caches.get(1);

      Fqn fqn = Fqn.fromString("/a/b");
      caches.get(0).put(fqn, "key", "value");

      assertEquals("value", caches.get(0).get(fqn, "key"));
      assertEquals("value", caches.get(1).get(fqn, "key"));

      // now make sure cache2 is in sync with cache1:
      caches.get(1).put(fqn, "key", "value");
      assertEquals("value", caches.get(1).get(fqn, "key"));
      assertEquals("value", caches.get(0).get(fqn, "key"));

      // now test the invalidation:
      caches.get(0).put(fqn, "key2", "value2");
      assertEquals("value2", caches.get(0).get(fqn, "key2"));
      assertEquals("value2", caches.get(1).get(fqn, "key2"));
      assertEquals("value", caches.get(0).get(fqn, "key"));
      assertEquals("value", caches.get(1).get(fqn, "key"));
   }

   public void testPessimisticTransactionalWithCacheLoader() throws Exception
   {
      List<CacheSPI<Object, Object>> caches = createCachesWithSharedCL(false);
      cache1 = caches.get(0);
      cache2 = caches.get(1);

      Fqn fqn = Fqn.fromString("/a/b");
      TransactionManager mgr = caches.get(0).getTransactionManager();
      assertNull("Should be null", caches.get(0).get(fqn, "key"));
      assertNull("Should be null", caches.get(1).get(fqn, "key"));
      mgr.begin();
      caches.get(0).put(fqn, "key", "value");
      assertEquals("value", caches.get(0).get(fqn, "key"));
      mgr.commit();
      assertEquals("value", caches.get(1).get(fqn, "key"));
      assertEquals("value", caches.get(0).get(fqn, "key"));

      mgr.begin();
      caches.get(0).put(fqn, "key2", "value2");
      assertEquals("value2", caches.get(0).get(fqn, "key2"));
      mgr.rollback();
      assertEquals("value", caches.get(1).get(fqn, "key"));
      assertEquals("value", caches.get(0).get(fqn, "key"));
      assertNull("Should be null", caches.get(0).get(fqn, "key2"));
      assertNull("Should be null", caches.get(1).get(fqn, "key2"));
   }

   public void testOptimisticWithCacheLoader() throws Exception
   {
      List<CacheSPI<Object, Object>> caches = createCachesWithSharedCL(true);
      cache1 = caches.get(0);
      cache2 = caches.get(1);

      Fqn fqn = Fqn.fromString("/a/b");
      TransactionManager mgr = caches.get(0).getTransactionManager();
      assertNull("Should be null", caches.get(0).get(fqn, "key"));
      assertNull("Should be null", caches.get(1).get(fqn, "key"));
      mgr.begin();
      caches.get(0).put(fqn, "key", "value");
      assertEquals("value", caches.get(0).get(fqn, "key"));
      assertNull("Should be null", caches.get(1).get(fqn, "key"));
      mgr.commit();
      assertEquals("value", caches.get(1).get(fqn, "key"));
      assertEquals("value", caches.get(0).get(fqn, "key"));

      mgr.begin();
      caches.get(0).put(fqn, "key2", "value2");
      assertEquals("value2", caches.get(0).get(fqn, "key2"));
      assertNull("Should be null", caches.get(1).get(fqn, "key2"));
      mgr.rollback();
      assertEquals("value", caches.get(1).get(fqn, "key"));
      assertEquals("value", caches.get(0).get(fqn, "key"));
      assertNull("Should be null", caches.get(0).get(fqn, "key2"));
      assertNull("Should be null", caches.get(1).get(fqn, "key2"));
   }

   public void testInvalidationWithRegionBasedMarshalling() throws Exception
   {
      doRegionBasedTest(false);
   }

   public void testInvalidationWithRegionBasedMarshallingOptimistic() throws Exception
   {
      doRegionBasedTest(true);
   }

   protected void doRegionBasedTest(boolean optimistic) throws Exception
   {
      List<CacheSPI<Object, Object>> caches = new ArrayList<CacheSPI<Object, Object>>();
      caches.add(createUnstartedCache(false));
      caches.add(createUnstartedCache(false));
      cache1 = caches.get(0);
      cache2 = caches.get(1);

      caches.get(0).getConfiguration().setUseRegionBasedMarshalling(true);
      caches.get(1).getConfiguration().setUseRegionBasedMarshalling(true);

      if (optimistic)
      {
         caches.get(0).getConfiguration().setNodeLockingScheme("OPTIMISTIC");
         caches.get(1).getConfiguration().setNodeLockingScheme("OPTIMISTIC");
      }

      caches.get(0).start();
      caches.get(1).start();

      TestingUtil.blockUntilViewsReceived(caches.toArray(new CacheSPI[0]), 5000);

      Fqn fqn = Fqn.fromString("/a/b");

      assertNull("Should be null", caches.get(0).getNode(fqn));
      assertNull("Should be null", caches.get(1).getNode(fqn));

      caches.get(0).put(fqn, "key", "value");
      assertEquals("expecting value", "value", caches.get(0).get(fqn, "key"));
      Node n = caches.get(1).getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");

      // now put in caches.get(1), should fire an eviction
      caches.get(1).put(fqn, "key", "value2");
      assertEquals("expecting value2", "value2", caches.get(1).get(fqn, "key"));
      n = caches.get(0).getNode(fqn);
      assertHasBeenInvalidated(n, "Should have been invalidated");
   }

   public void testDeleteNonExistentPessimistic() throws Exception
   {
      deleteNonExistentTest(false);
   }

   /**
    * Test for JBCACHE-1297
    *
    * @throws Exception
    */
   public void testDeleteNonExistentOptimistic() throws Exception
   {
      deleteNonExistentTest(true);
   }

   private void deleteNonExistentTest(boolean optimistic) throws Exception
   {
      List<CacheSPI<Object, Object>> caches = new ArrayList<CacheSPI<Object, Object>>();
      caches.add(createUnstartedCache(optimistic));
      caches.add(createUnstartedCache(optimistic));
      cache1 = caches.get(0);
      cache2 = caches.get(1);

      cache1.start();
      cache2.start();

      TestingUtil.blockUntilViewsReceived(caches.toArray(new CacheSPI[0]), 5000);

      Fqn fqn = Fqn.fromString("/a/b");

      assertNull("Should be null", cache1.getNode(fqn));
      assertNull("Should be null", cache2.getNode(fqn));

      cache1.putForExternalRead(fqn, "key", "value");

      assertEquals("value", cache1.getNode(fqn).get("key"));
      assertNull("Should be null", cache2.getNode(fqn));

      // OK, here's the real test
      TransactionManager tm = cache2.getTransactionManager();
      tm.begin();
      try
      {
         // Remove a node that doesn't exist in cache2
         cache2.removeNode(fqn);
         tm.commit();
      }
      catch (Exception e)
      {
         String msg = "Unable to remove non-existent node " + fqn;
         log.error(msg, e);
         fail(msg + " -- " + e);
      }

      assertHasBeenInvalidated(cache1.getNode(fqn), "Should have been invalidated");
      assertNull("Should be null", cache2.getNode(fqn));
   }

   /**
    * Test for JBCACHE-1298.
    *
    * @throws Exception
    */
   public void testAddOfDeletedNonExistent() throws Exception
   {
      cache1 = createCache(true);
      cache2 = createCache(true);

      TestingUtil.blockUntilViewsReceived(5000, cache1, cache2);

      Fqn fqn = Fqn.fromString("/a/b");

      assertNull("Should be null", cache1.getNode(fqn));
      assertNull("Should be null", cache2.getNode(fqn));

      // OK, here's the real test
      TransactionManager tm = cache2.getTransactionManager();
      tm.begin();
      try
      {
         // Remove a node that doesn't exist in cache2
         cache2.removeNode(fqn);
         tm.commit();
      }
      catch (Exception e)
      {
         String msg = "Unable to remove non-existent node " + fqn;
         log.error(msg, e);
         fail(msg + " -- " + e);
      }

      // Actually, it shouldn't have been invalidated, should be null
      // But, this assertion will pass if it is null, and we want
      assertHasBeenInvalidated(cache1.getNode(fqn), "Should have been invalidated");
      assertNull("Should be null", cache2.getNode(fqn));

      cache1.getInvocationContext().getOptionOverrides().setDataVersion(new DefaultDataVersion());
      cache1.put(fqn, "key", "value");

      assertEquals("value", cache1.getNode(fqn).get("key"));
      assertHasBeenInvalidated(cache2.getNode(fqn), "Should have been invalidated");
   }

   protected CacheSPI<Object, Object> createUnstartedCache(boolean optimistic) throws Exception
   {
      CacheSPI<Object, Object> cache = (CacheSPI<Object, Object>) new DefaultCacheFactory().createCache(false);
      cache.getConfiguration().setClusterName("MyCluster");
      cache.getConfiguration().setStateRetrievalTimeout(3000);
      cache.getConfiguration().setCacheMode(Configuration.CacheMode.INVALIDATION_SYNC);
      if (optimistic) cache.getConfiguration().setNodeLockingScheme("OPTIMISTIC");
      cache.getConfiguration().setTransactionManagerLookupClass("org.jboss.cache.transaction.DummyTransactionManagerLookup");
      toClean.add(cache);
      return cache;
   }

   protected CacheSPI<Object, Object> createCache(boolean optimistic) throws Exception
   {
      CacheSPI<Object, Object> cache = createUnstartedCache(optimistic);
      cache.start();
      toClean.add(cache);
      return cache;
   }

   protected List<CacheSPI<Object, Object>> createCachesWithSharedCL(boolean optimistic) throws Exception
   {
      List<CacheSPI<Object, Object>> caches = new ArrayList<CacheSPI<Object, Object>>();
      caches.add(createUnstartedCache(optimistic));
      caches.add(createUnstartedCache(optimistic));

      caches.get(0).getConfiguration().setCacheLoaderConfig(getCacheLoaderConfig());
      caches.get(1).getConfiguration().setCacheLoaderConfig(getCacheLoaderConfig());

      caches.get(0).start();
      caches.get(1).start();
      toClean.addAll(caches);
      return caches;
   }


   protected CacheLoaderConfig getCacheLoaderConfig() throws Exception
   {
      String xml = "            <config>\n" +
            "                <shared>shared</shared>\n" +
            "                <passivation>false</passivation>\n" +
            "                <preload></preload>\n" +
            "                <cacheloader>\n" +
            "                    <class>org.jboss.cache.loader.DummySharedInMemoryCacheLoader</class>\n" +
            "                    <async>false</async>\n" +
            "                    <fetchPersistentState>false</fetchPersistentState>\n" +
            "                    <ignoreModifications>false</ignoreModifications>\n" +
            "                </cacheloader>\n" +
            "                \n" +
            "            </config>";
      Element element = XmlHelper.stringToElement(xml);
      return XmlConfigurationParser.parseCacheLoaderConfig(element);
   }

   protected void assertHasBeenInvalidated(Node n, String message)
   {
      // depending on how n was retrieved!
      if (n == null)
      {
         assert true : message;
      }
      else
      {
         assert !n.isValid() : message;
      }
   }
}
