/*
// This software is subject to the terms of the Eclipse Public License v1.0
// Agreement, available at the following URL:
// http://www.eclipse.org/legal/epl-v10.html.
// You must accept the terms of that agreement to use this software.
//
// Copyright (c) 2002-2015 Pentaho Corporation.
// All Rights Reserved.
*/
package mondrian.rolap.cache;

import mondrian.olap.QueryCanceledException;
import mondrian.olap.Util;
import mondrian.rolap.BitKey;
import mondrian.rolap.RolapUtil;
import mondrian.rolap.agg.*;
import mondrian.server.Execution;
import mondrian.spi.*;
import mondrian.util.*;

import org.apache.log4j.Logger;

import java.io.PrintWriter;
import java.sql.Statement;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;


/**
 * Data structure that identifies which segments contain cells.
 *
 * <p>Not thread safe.</p>
 *
 * @author Julian Hyde
 */
public class SegmentCacheIndexImpl implements SegmentCacheIndex {

    private static final Logger LOGGER =
        Logger.getLogger(SegmentCacheIndexImpl.class);

    private final Map<List, List<SegmentHeader>> bitkeyMap =
        new HashMap<List, List<SegmentHeader>>();

    /**
     * The fact map allows us to spot quickly which
     * segments have facts relating to a given header.
     */
    private final Map<List, FactInfo> factMap =
        new HashMap<List, FactInfo>();

    /**
     * The fuzzy fact map allows us to spot quickly which
     * segments have facts relating to a given header, but doesn't
     * consider the compound predicates in the key. This allows
     * flush operations to be consistent.
     */
    // TODO Get rid of the fuzzy map once we have a way to parse
    // compound predicates into rich objects that can be serialized
    // as part of the SegmentHeader.
    private final Map<List, FuzzyFactInfo> fuzzyFactMap =
        new HashMap<List, FuzzyFactInfo>();

    private final Map<SegmentHeader, HeaderInfo> headerMap =
        new HashMap<SegmentHeader, HeaderInfo>();

    private final Thread thread;

    /**
     * Creates a SegmentCacheIndexImpl.
     *
     * @param thread Thread that must be used to execute commands.
     */
    public SegmentCacheIndexImpl(Thread thread) {
        this.thread = thread;
        assert thread != null;
    }

    public static List makeConverterKey(SegmentHeader header) {
        return Arrays.asList(
            header.schemaName,
            header.schemaChecksum,
            header.cubeName,
            header.rolapStarFactTableName,
            header.measureName,
            header.compoundPredicates);
    }

    public static List makeConverterKey(CellRequest request, AggregationKey key)
    {
        return Arrays.asList(
            request.getMeasure().getStar().getSchema().getName(),
            request.getMeasure().getStar().getSchema().getChecksum(),
            request.getMeasure().getCubeName(),
            request.getMeasure().getStar().getFactTable().getAlias(),
            request.getMeasure().getName(),
            request.getCompoundPredicateStrings());
    }

    public List<SegmentHeader> locate(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String measureName,
        String rolapStarFactTableName,
        BitKey constrainedColsBitKey,
        Map<String, Comparable> coordinates,
        List<String> compoundPredicates)
    {
        checkThread();

        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace(
                "SegmentCacheIndexImpl("
                + System.identityHashCode(this)
                + ")locate:"
                + "\nschemaName:" + schemaName
                + "\nschemaChecksum:" + schemaChecksum
                + "\ncubeName:" + cubeName
                + "\nmeasureName:" + measureName
                + "\nrolapStarFactTableName:" + rolapStarFactTableName
                + "\nconstrainedColsBitKey:" + constrainedColsBitKey
                + "\ncoordinates:" + coordinates
                + "\ncompoundPredicates:" + compoundPredicates);
        }

        List<SegmentHeader> list = Collections.emptyList();
        final List starKey =
            makeBitkeyKey(
                schemaName,
                schemaChecksum,
                cubeName,
                rolapStarFactTableName,
                constrainedColsBitKey,
                measureName,
                compoundPredicates);
        final List<SegmentHeader> headerList = bitkeyMap.get(starKey);
        if (headerList == null) {
            LOGGER.trace(
                "SegmentCacheIndexImpl("
                + System.identityHashCode(this)
                + ").locate:NOMATCH");
            return Collections.emptyList();
        }
        for (SegmentHeader header : headerList) {
            if (matches(header, coordinates, compoundPredicates)) {
                // Be lazy. Don't allocate a list unless there is at least one
                // entry.
                if (list.isEmpty()) {
                    list = new ArrayList<SegmentHeader>();
                }
                list.add(header);
            }
        }
        if (LOGGER.isTraceEnabled()) {
            final StringBuilder sb =
                new StringBuilder(
                    "SegmentCacheIndexImpl("
                    + System.identityHashCode(this)
                    + ").locate:MATCH");
            for (SegmentHeader header : list) {
                sb.append("\n");
                sb.append(header.toString());
            }
            LOGGER.trace(sb.toString());
        }
        return list;
    }

    public void add(
        SegmentHeader header,
        SegmentBuilder.SegmentConverter converter,
        boolean loading)
    {
        checkThread();

        LOGGER.debug(
            "SegmentCacheIndexImpl("
            + System.identityHashCode(this)
            + ").add:\n"
            + header.toString());

        HeaderInfo headerInfo = headerMap.get(header);
        if (headerInfo == null) {
            headerInfo = new HeaderInfo();
            if (loading) {
                // We are currently loading this segment. It isnt' in cache.
                // We put a slot into which the data will become available.
                headerInfo.slot = new SlotFuture<SegmentBody>();
            }
            headerMap.put(header, headerInfo);
        }

        final List bitkeyKey = makeBitkeyKey(header);
        List<SegmentHeader> headerList = bitkeyMap.get(bitkeyKey);
        if (headerList == null) {
            headerList = new ArrayList<SegmentHeader>();
            bitkeyMap.put(bitkeyKey, headerList);
        }
        if (!headerList.contains(header)) {
            headerList.add(header);
        }

        final List factKey = makeFactKey(header);
        FactInfo factInfo = factMap.get(factKey);
        if (factInfo == null) {
            factInfo = new FactInfo();
            factMap.put(factKey, factInfo);
        }
        if (!factInfo.headerList.contains(header)) {
            factInfo.headerList.add(header);
        }
        if (!factInfo.bitkeyPoset
            .contains(header.getConstrainedColumnsBitKey()))
        {
            factInfo.bitkeyPoset.add(header.getConstrainedColumnsBitKey());
        }
        if (converter != null) {
            factInfo.converter = converter;
        }

        final List fuzzyFactKey = makeFuzzyFactKey(header);
        FuzzyFactInfo fuzzyFactInfo = fuzzyFactMap.get(fuzzyFactKey);
        if (fuzzyFactInfo == null) {
            fuzzyFactInfo = new FuzzyFactInfo();
            fuzzyFactMap.put(fuzzyFactKey, fuzzyFactInfo);
        }
        if (!fuzzyFactInfo.headerList.contains(header)) {
            fuzzyFactInfo.headerList.add(header);
        }
    }

    public void update(
        SegmentHeader oldHeader,
        SegmentHeader newHeader)
    {
        checkThread();

        LOGGER.trace(
            "SegmentCacheIndexImpl.update: Updating header from:\n"
            + oldHeader.toString()
            + "\n\nto\n\n"
            + newHeader.toString());
        final HeaderInfo headerInfo = headerMap.get(oldHeader);
        headerMap.remove(oldHeader);
        headerMap.put(newHeader, headerInfo);

        final List oldBitkeyKey = makeBitkeyKey(oldHeader);
        List<SegmentHeader> headerList = bitkeyMap.get(oldBitkeyKey);
        headerList.remove(oldHeader);
        headerList.add(newHeader);

        final List oldFactKey = makeFactKey(oldHeader);
        final FactInfo factInfo = factMap.get(oldFactKey);
        factInfo.headerList.remove(oldHeader);
        factInfo.headerList.add(newHeader);

        final List oldFuzzyFactKey = makeFuzzyFactKey(oldHeader);
        final FuzzyFactInfo fuzzyFactInfo = fuzzyFactMap.get(oldFuzzyFactKey);
        fuzzyFactInfo.headerList.remove(oldHeader);
        fuzzyFactInfo.headerList.add(newHeader);
    }

    public void loadSucceeded(SegmentHeader header, SegmentBody body) {
        checkThread();

        final HeaderInfo headerInfo = headerMap.get(header);
        assert headerInfo != null
            : "segment header " + header.getUniqueID() + " is missing";
        if (!headerInfo.slot.isDone()) {
            headerInfo.slot.put(body);
        }
        if (headerInfo.removeAfterLoad) {
            remove(header);
        }
        // Cleanup the HeaderInfo
        headerInfo.stmt = null;
        headerInfo.clients.clear();
    }

    public void loadFailed(SegmentHeader header, Throwable throwable) {
        checkThread();

        final HeaderInfo headerInfo = headerMap.get(header);
        if (headerInfo == null) {
            LOGGER.trace("loadFailed: Missing header " + header);
            return;
        }
        assert headerInfo.slot != null
            : "segment header " + header.getUniqueID() + " is not loading";
        headerInfo.slot.fail(throwable);
        remove(header);
        // Cleanup the HeaderInfo
        headerInfo.stmt = null;
        headerInfo.clients.clear();
    }

    public void remove(SegmentHeader header) {
        checkThread();

        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace(
                "SegmentCacheIndexImpl("
                + System.identityHashCode(this)
                + ").remove:\n"
                + header.toString(),
                new Throwable("Removal."));
        } else {
            LOGGER.debug(
                "SegmentCacheIndexImpl.remove:\n"
                + header.toString());
        }

        final HeaderInfo headerInfo = headerMap.get(header);
        if (headerInfo == null) {
            LOGGER.debug(
                "SegmentCacheIndexImpl("
                + System.identityHashCode(this)
                + ").remove:UNKNOWN HEADER");
            return;
        }
        if (headerInfo.slot != null && !headerInfo.slot.isDone()) {
            // Cannot remove while load is pending; flag for removal after load
            headerInfo.removeAfterLoad = true;
            LOGGER.debug(
                "SegmentCacheIndexImpl("
                + System.identityHashCode(this)
                + ").remove:DEFFERED");
            return;
        }

        headerMap.remove(header);

        final List factKey = makeFactKey(header);
        final FactInfo factInfo = factMap.get(factKey);
        if (factInfo != null) {
            factInfo.headerList.remove(header);
            factInfo.bitkeyPoset.remove(header.getConstrainedColumnsBitKey());
            if (factInfo.headerList.size() == 0) {
                factMap.remove(factKey);
            }
        }

        final List fuzzyFactKey = makeFuzzyFactKey(header);
        final FuzzyFactInfo fuzzyFactInfo = fuzzyFactMap.get(fuzzyFactKey);
        if (fuzzyFactInfo != null) {
            fuzzyFactInfo.headerList.remove(header);
            if (fuzzyFactInfo.headerList.size() == 0) {
                fuzzyFactMap.remove(fuzzyFactKey);
            }
        }

        final List bitkeyKey = makeBitkeyKey(header);
        final List<SegmentHeader> headerList = bitkeyMap.get(bitkeyKey);
        headerList.remove(header);
        if (headerList.size() == 0) {
            bitkeyMap.remove(bitkeyKey);
        }
    }

    private void checkThread() {
        assert thread == Thread.currentThread()
            : "expected " + thread + ", but was " + Thread.currentThread();
    }

    public static boolean matches(
        SegmentHeader header,
        Map<String, Comparable> coords,
        List<String> compoundPredicates)
    {
        if (!header.compoundPredicates.equals(compoundPredicates)) {
            return false;
        }
        for (Map.Entry<String, Comparable> entry : coords.entrySet()) {
            // Check if the segment explicitly excludes this coordinate.
            final SegmentColumn excludedColumn =
                header.getExcludedRegion(entry.getKey());
            if (excludedColumn != null) {
                final SortedSet<Comparable> values =
                    excludedColumn.getValues();
                if (values == null || values.contains(entry.getValue())) {
                    return false;
                }
            }
            // Check if the dimensionality of the segment intersects
            // with the coordinate.
            final SegmentColumn constrainedColumn =
                header.getConstrainedColumn(entry.getKey());
            if (constrainedColumn == null) {
                // One of the required column/value pairs is not a constraining
                // column for the header. This will not happen if the header
                // has been acquired from bitkeyMap, but may happen if a list
                // of mixed-dimensionality headers is being scanned.
                return false;
            }
            final SortedSet<Comparable> values =
                constrainedColumn.getValues();
            if (values != null
                && !values.contains(entry.getValue()))
            {
                return false;
            }
        }
        return true;
    }

    public List<SegmentHeader> intersectRegion(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String measureName,
        String rolapStarFactTableName,
        SegmentColumn[] region)
    {
        checkThread();

        final List factKey = makeFuzzyFactKey(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName);
        final FuzzyFactInfo factInfo = fuzzyFactMap.get(factKey);
        List<SegmentHeader> list = Collections.emptyList();
        if (factInfo == null) {
            return list;
        }
        for (SegmentHeader header : factInfo.headerList) {
            // Don't return stale segments.
            if (headerMap.get(header).removeAfterLoad) {
                continue;
            }
            if (intersects(header, region)) {
                // Be lazy. Don't allocate a list unless there is at least one
                // entry.
                if (list.isEmpty()) {
                    list = new ArrayList<SegmentHeader>();
                }
                list.add(header);
            }
        }
        return list;
    }

    private boolean intersects(
        SegmentHeader header,
        SegmentColumn[] region)
    {
        // most selective condition first
        if (region.length == 0) {
            return true;
        }
        for (SegmentColumn regionColumn : region) {
            final SegmentColumn headerColumn =
                header.getConstrainedColumn(regionColumn.getColumnExpression());
            if (headerColumn == null) {
                // If the segment header doesn't contain a column specified
                // by the region, then it always implicitly intersects.
                // This allows flush operations to be valid.
                return true;
            }
            final SortedSet<Comparable> regionValues =
                regionColumn.getValues();
            final SortedSet<Comparable> headerValues =
                headerColumn.getValues();
            if (headerValues == null || regionValues == null) {
                // This is a wildcard, so it always intersects.
                return true;
            }
            for (Comparable myValue : regionValues) {
                if (headerValues.contains(myValue)) {
                    return true;
                }
            }
        }
        return false;
    }

    public void printCacheState(PrintWriter pw) {
        checkThread();
        final List<List<SegmentHeader>> values =
            new ArrayList<List<SegmentHeader>>(
                bitkeyMap.values());
        Collections.sort(
            values,
            new Comparator<List<SegmentHeader>>() {
                public int compare(
                    List<SegmentHeader> o1,
                    List<SegmentHeader> o2)
                {
                    if (o1.size() == 0) {
                        return -1;
                    }
                    if (o2.size() == 0) {
                        return 1;
                    }
                    return o1.get(0).getUniqueID()
                        .compareTo(o2.get(0).getUniqueID());
                }
            });
        for (List<SegmentHeader> key : values) {
            final List<SegmentHeader> headerList =
                new ArrayList<SegmentHeader>(key);
            Collections.sort(
                headerList,
                new Comparator<SegmentHeader>() {
                    public int compare(SegmentHeader o1, SegmentHeader o2) {
                        return o1.getUniqueID().compareTo(o2.getUniqueID());
                    }
                });
            for (SegmentHeader header : headerList) {
                pw.println(header.getDescription());
            }
        }
    }

    public Future<SegmentBody> getFuture(Execution exec, SegmentHeader header) {
        checkThread();
        HeaderInfo hi = headerMap.get(header);
        if (!hi.clients.contains(exec)) {
            hi.clients.add(exec);
        }
        return hi.slot;
    }

    public void linkSqlStatement(SegmentHeader header, Statement stmt) {
        checkThread();
        headerMap.get(header).stmt = stmt;
    }

    public boolean contains(SegmentHeader header) {
        return headerMap.containsKey(header);
    }

    public void cancel(Execution exec) {
        checkThread();
        List<SegmentHeader> toRemove = new ArrayList<SegmentHeader>();
        for (Entry<SegmentHeader, HeaderInfo> entry : headerMap.entrySet()) {
            if (entry.getValue().clients.remove(exec)) {
                if (entry.getValue().slot != null
                    && !entry.getValue().slot.isDone()
                    && entry.getValue().clients.isEmpty())
                {
                    toRemove.add(entry.getKey());
                }
            }
        }
        // Make sure to cleanup the orphaned segments.
        for (SegmentHeader header : toRemove) {
            final Statement stmt = headerMap.get(header).stmt;
            loadFailed(
                header,
                new QueryCanceledException(
                    "Canceling due to an absence of interested parties."));
            // We only want to cancel the statement, but we can't close it.
            // Some drivers will not notice the interruption flag on their
            // own thread before a considerable time has passed. If we were
            // using a pooling layer, calling close() would make the
            // underlying connection available again, despite the first
            // statement still being processed. Some drivers will fail
            // there. It is therefore important to close and release the
            // resources on the proper thread, namely, the thread which
            // runs the actual statement.
            Util.cancelStatement(stmt);
        }
    }

    public SegmentBuilder.SegmentConverter getConverter(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String rolapStarFactTableName,
        String measureName,
        List<String> compoundPredicates)
    {
        checkThread();

        final List factKey = makeFactKey(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName,
            compoundPredicates);
        final FactInfo factInfo = factMap.get(factKey);
        if (factInfo == null) {
            return null;
        }
        return factInfo.converter;
    }

    public void setConverter(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String rolapStarFactTableName,
        String measureName,
        List<String> compoundPredicates,
        SegmentBuilder.SegmentConverter converter)
    {
        checkThread();

        final List factKey = makeFactKey(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName,
            compoundPredicates);
        final FactInfo factInfo = factMap.get(factKey);
        assert factInfo != null : "should have called 'add' first";
        if (factInfo == null) {
            return;
        }
        factInfo.converter = converter;
    }

    private List makeBitkeyKey(SegmentHeader header) {
        return makeBitkeyKey(
            header.schemaName,
            header.schemaChecksum,
            header.cubeName,
            header.rolapStarFactTableName,
            header.constrainedColsBitKey,
            header.measureName,
            header.compoundPredicates);
    }

    private List makeBitkeyKey(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String rolapStarFactTableName,
        BitKey constrainedColsBitKey,
        String measureName,
        List<String> compoundPredicates)
    {
        return Arrays.asList(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            constrainedColsBitKey,
            measureName,
            compoundPredicates);
    }

    private List makeFactKey(SegmentHeader header) {
        return makeFactKey(
            header.schemaName,
            header.schemaChecksum,
            header.cubeName,
            header.rolapStarFactTableName,
            header.measureName,
            header.compoundPredicates);
    }

    private List makeFactKey(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String rolapStarFactTableName,
        String measureName,
        List<String> compoundPredicates)
    {
        return Arrays.asList(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName,
            compoundPredicates);
    }

    private List makeFuzzyFactKey(SegmentHeader header) {
        return makeFuzzyFactKey(
            header.schemaName,
            header.schemaChecksum,
            header.cubeName,
            header.rolapStarFactTableName,
            header.measureName);
    }

    private List makeFuzzyFactKey(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String rolapStarFactTableName,
        String measureName)
    {
        return Arrays.asList(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName);
    }

    public List<List<SegmentHeader>> findRollupCandidates(
        String schemaName,
        ByteString schemaChecksum,
        String cubeName,
        String measureName,
        String rolapStarFactTableName,
        BitKey constrainedColsBitKey,
        Map<String, Comparable> coordinates,
        List<String> compoundPredicates)
    {
        final List factKey = makeFactKey(
            schemaName,
            schemaChecksum,
            cubeName,
            rolapStarFactTableName,
            measureName,
            compoundPredicates);
        final FactInfo factInfo = factMap.get(factKey);
        if (factInfo == null) {
            return Collections.emptyList();
        }

        // Iterate over all dimensionalities that are a superset of the desired
        // columns and for which a segment is known to exist.
        //
        // It helps that getAncestors returns dimensionalities with fewer bits
        // set first. These will contain fewer cells, and therefore be less
        // effort to roll up.

        final List<List<SegmentHeader>> list =
            new ArrayList<List<SegmentHeader>>();
        final List<BitKey> ancestors =
            factInfo.bitkeyPoset.getAncestors(constrainedColsBitKey);
        for (BitKey bitKey : ancestors) {
            final List bitkeyKey = makeBitkeyKey(
                schemaName,
                schemaChecksum,
                cubeName,
                rolapStarFactTableName,
                bitKey,
                measureName,
                compoundPredicates);
            final List<SegmentHeader> headers = bitkeyMap.get(bitkeyKey);
            assert headers != null : "bitkeyPoset / bitkeyMap inconsistency";

            // For columns that are still present after roll up, make sure that
            // the required value is in the range covered by the segment.
            // Of the columns that are being aggregated away, are all of
            // them wildcarded? If so, this segment is a match. If not, we
            // will need to combine with other segments later.
            findRollupCandidatesAmong(coordinates, list, headers);
        }
        return list;
    }

    /**
     * Finds rollup candidates among a list of headers with the same
     * dimensionality.
     *
     * <p>For each column that is being aggregated away, we need to ensure that
     * we have all values of that column. If the column is wildcarded, it's
     * easy. For example, if we wish to roll up to create Segment1:</p>
     *
     * <pre>Segment1(Year=1997, MaritalStatus=*)</pre>
     *
     * <p>then clearly Segment2:</p>
     *
     * <pre>Segment2(Year=1997, MaritalStatus=*, Gender=*, Nation=*)</pre>
     *
     * <p>has all gender and Nation values. If the values are specified as a
     * list:</p>
     *
     * <pre>Segment3(Year=1997, MaritalStatus=*, Gender={M, F}, Nation=*)</pre>
     *
     * <p>then we need to check the metadata. We see that Gender has two
     * distinct values in the database, and we have two values, therefore we
     * have all of them.</p>
     *
     * <p>What if we have multiple non-wildcard columns? Consider:</p>
     *
     * <pre>
     *     Segment4(Year=1997, MaritalStatus=*, Gender={M},
                    Nation={Mexico, USA})
     *     Segment5(Year=1997, MaritalStatus=*, Gender={F},
                    Nation={USA})
     *     Segment6(Year=1997, MaritalStatus=*, Gender={F, M},
                    Nation={Canada, Mexico, Honduras, Belize})
     * </pre>
     *
     * <p>The problem is similar to finding whether a collection of rectangular
     * regions covers a rectangle (or, generalizing to n dimensions, an
     * n-cube). Or better, find a minimal collection of regions.</p>
     *
     * <p>Our algorithm solves it by iterating over all combinations of values.
     * Those combinations are exponential in theory, but tractible in practice,
     * using the following trick. The algorithm reduces the number of
     * combinations by looking for values that are always treated the same. In
     * the above, Canada, Honduras and Belize are always treated the same, so to
     * prove covering, it is sufficient to prove that all combinations involving
     * Canada are covered.</p>
     *
     * @param coordinates Coordinates
     * @param list List to write candidates to
     * @param headers Headers of candidate segments
     */
    private void findRollupCandidatesAmong(
        Map<String, Comparable> coordinates,
        List<List<SegmentHeader>> list,
        List<SegmentHeader> headers)
    {
        final List<Pair<SegmentHeader, List<SegmentColumn>>> matchingHeaders =
            new ArrayList<Pair<SegmentHeader, List<SegmentColumn>>>();
        headerLoop:
        for (SegmentHeader header : headers) {
            // Skip headers that have exclusions.
            //
            // TODO: This is a bit harsh.
            if (!header.getExcludedRegions().isEmpty()) {
                continue;
            }

            List<SegmentColumn> nonWildcards =
                new ArrayList<SegmentColumn>();
            for (SegmentColumn column : header.getConstrainedColumns()) {
                final SegmentColumn constrainedColumn =
                    header.getConstrainedColumn(column.columnExpression);

                // REVIEW: How are null key values represented in coordinates?
                // Assuming that they are represented by null ref.
                if (coordinates.containsKey(column.columnExpression)) {
                    // Matching column. Will not be aggregated away. Needs
                    // to be in range.
                    Comparable value =
                        coordinates.get(column.columnExpression);
                    if (value == null) {
                        value = RolapUtil.sqlNullValue;
                    }
                    if (constrainedColumn.values != null
                        && !constrainedColumn.values.contains(value))
                    {
                        continue headerLoop;
                    }
                } else {
                    // Non-matching column. Will be aggregated away. Needs
                    // to be wildcarded (or some more complicated conditions
                    // to be dealt with later).
                    if (constrainedColumn.values != null) {
                        nonWildcards.add(constrainedColumn);
                    }
                }
            }

            if (nonWildcards.isEmpty()) {
                list.add(Collections.singletonList(header));
            } else {
                matchingHeaders.add(Pair.of(header, nonWildcards));
            }
        }

        // Find combinations of segments that can roll up. Need at least two.
        if (matchingHeaders.size() < 2) {
            return;
        }

        // Collect the list of non-wildcarded columns.
        final List<SegmentColumn> columnList = new ArrayList<SegmentColumn>();
        final List<String> columnNameList = new ArrayList<String>();
        for (Pair<SegmentHeader, List<SegmentColumn>> pair : matchingHeaders) {
            for (SegmentColumn column : pair.right) {
                if (!columnNameList.contains(column.columnExpression)) {
                    final int valueCount = column.getValueCount();
                    if (valueCount <= 0) {
                        // Impossible to safely roll up. If we don't know the
                        // number of values, we don't know that we have all of
                        // them.
                        return;
                    }
                    columnList.add(column);
                    columnNameList.add(column.columnExpression);
                }
            }
        }

        // Gather known values of each column. For each value, remember which
        // segments refer to it.
        final List<List<Comparable>> valueLists =
            new ArrayList<List<Comparable>>();
        for (SegmentColumn column : columnList) {
            // For each value, which equivalence class it belongs to.
            final SortedMap<Comparable, BitSet> valueMap =
                new TreeMap<Comparable, BitSet>(RolapUtil.ROLAP_COMPARATOR);

            int h = -1;
            for (SegmentHeader header : Pair.leftIter(matchingHeaders)) {
                ++h;
                final SegmentColumn column1 =
                    header.getConstrainedColumn(
                        column.columnExpression);
                if (column1.getValues() == null) {
                    // Wildcard. Mark all values as present.
                    for (Entry<Comparable, BitSet> entry : valueMap.entrySet())
                    {
                        for (int pos = 0;
                            pos < entry.getValue().cardinality();
                            pos++)
                        {
                            entry.getValue().set(pos);
                        }
                    }
                } else {
                    for (Comparable value : column1.getValues()) {
                        BitSet bitSet = valueMap.get(value);
                        if (bitSet == null) {
                            bitSet = new BitSet();
                            valueMap.put(value, bitSet);
                        }
                        bitSet.set(h);
                    }
                }
            }

            // Is the number of values discovered equal to the known cardinality
            // of the column? If not, we can't cover the space.
            if (valueMap.size() < column.valueCount) {
                return;
            }

            // Build equivalence sets of values. These group together values
            // that are used identically in segments.
            //
            // For instance, given segments Sx over column c,
            //
            // S1: c = {1, 2, 3, 4}
            // S2: c = {3, 4, 5}
            // S3: c = {3, 6, 7, 8}
            //
            // the equivalence classes are:
            //
            // E1 = {1, 2} used in {S1}
            // E2 = {3} used in {S1, S2, S3}
            // E3 = {4} used in {S1, S2}
            // E4 = {6, 7, 8} used in {S3}
            //
            // The equivalence classes reduce the size of the search space. (In
            // this case, from 8 values to 4 classes.) We can use any value in a
            // class to stand for all values.
            final Map<BitSet, Comparable> eqclassPrimaryValues =
                new HashMap<BitSet, Comparable>();
            for (Map.Entry<Comparable, BitSet> entry : valueMap.entrySet()) {
                final BitSet bitSet = entry.getValue();
                if (!eqclassPrimaryValues.containsKey(bitSet)) {
                    final Comparable value = entry.getKey();
                    eqclassPrimaryValues.put(bitSet, value);
                }
            }
            valueLists.add(
                new ArrayList<Comparable>(
                    eqclassPrimaryValues.values()));
        }

        // Iterate over every combination of values, and make sure that some
        // segment can satisfy each.
        //
        // TODO: A greedy algorithm would probably be better. Rather than adding
        // the first segment that contains a particular value combination, add
        // the segment that contains the most value combinations that we are are
        // not currently covering.
        final CartesianProductList<Comparable> tuples =
            new CartesianProductList<Comparable>(valueLists);
        final List<SegmentHeader> usedSegments = new ArrayList<SegmentHeader>();
        final List<SegmentHeader> unusedSegments =
            new ArrayList<SegmentHeader>(Pair.left(matchingHeaders));
        tupleLoop:
        for (List<Comparable> tuple : tuples) {
            // If the value combination is handled by one of the used segments,
            // great!
            for (SegmentHeader segment : usedSegments) {
                if (contains(segment, tuple, columnNameList)) {
                    continue tupleLoop;
                }
            }
            // Does one of the unused segments contain it? Use the first one we
            // find.
            for (SegmentHeader segment : unusedSegments) {
                if (contains(segment, tuple, columnNameList)) {
                    unusedSegments.remove(segment);
                    usedSegments.add(segment);
                    continue tupleLoop;
                }
            }
            // There was a value combination not contained in any of the
            // segments. Fail.
            return;
        }
        list.add(usedSegments);
    }

    private boolean contains(
        SegmentHeader segment,
        List<Comparable> values,
        List<String> columns)
    {
        for (int i = 0; i < columns.size(); i++) {
            String columnName = columns.get(i);
            final SegmentColumn column =
                segment.getConstrainedColumn(columnName);
            final SortedSet<Comparable> valueSet = column.getValues();
            if (valueSet != null && !valueSet.contains(values.get(i))) {
                return false;
            }
        }
        return true;
    }

    private static class FactInfo {
        private static final PartiallyOrderedSet.Ordering<BitKey> ORDERING =
            new PartiallyOrderedSet.Ordering<BitKey>() {
                public boolean lessThan(BitKey e1, BitKey e2) {
                    return e2.isSuperSetOf(e1);
                }
            };

        private final List<SegmentHeader> headerList =
            new ArrayList<SegmentHeader>();

        private final PartiallyOrderedSet<BitKey> bitkeyPoset =
            new PartiallyOrderedSet<BitKey>(ORDERING);

        private SegmentBuilder.SegmentConverter converter;

        FactInfo() {
        }
    }

    private static class FuzzyFactInfo {
        private final List<SegmentHeader> headerList =
            new ArrayList<SegmentHeader>();

        FuzzyFactInfo() {
        }
    }

    /**
     * A private class that we use in the index to track who was interested in
     * which headers, the SQL statement that is populating it and a future
     * object which we pass to clients.
     */
    private static class HeaderInfo {
        /**
         * The SQL statement populating this header.
         * Will be null until the SQL thread calls us back to register it.
         */
        private Statement stmt;
        /**
         * The future object to pass on to clients.
         */
        private SlotFuture<SegmentBody> slot;
        /**
         * A list of clients interested in this segment.
         */
        private final List<Execution> clients =
            new CopyOnWriteArrayList<Execution>();
        /**
         * Whether this segment is already considered stale and must
         * be deleted after it is done loading. This can happen
         * when flushing.
         */
        private boolean removeAfterLoad;
    }
}

// End SegmentCacheIndexImpl.java
