SET search_path TO documentdb_core,documentdb_api,documentdb_api_catalog,documentdb_api_internal;

-- Insert data
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 1, "item_name" : "shirt", "price" : 12, "order_quantity" : 2 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases','{ "_id" : 2, "item_name" : "pants", "price" : 20, "order_quantity" : 1 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 3, "item_name" : "hat", "price" : 10, "order_quantity" : 5 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 4, "item_name" : ["shirt", "hat", "pants"], "price" : 10, "order_quantity" : 5 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 5}', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 6, "item_name" : {"a": "x", "b" : 1, "c" : [1, 2, 3]} }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases',' { "_id" : 7, "item_name" : [{"a": { "b" : 1}}, [1, 2, 3], 1, "x"] }', NULL);

SELECT documentdb_api.insert_one('db','catalog_items',' { "_id" : 1, "item_code" : "shirt", "product_description": "product 1", "stock_quantity" : 120 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items',' { "_id" : 11, "item_code" : "shirt", "product_description": "product 1", "stock_quantity" : 240 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 2, "item_code" : "hat", "product_description": "product 2", "stock_quantity" : 80 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 3, "item_code" : "shoes", "product_description": "product 3", "stock_quantity" : 60 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 4, "item_code" : "pants", "product_description": "product 4", "stock_quantity" : 70 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 5, "item_code" : null, "product_description": "product 4", "stock_quantity" : 70 }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 6, "item_code" :  {"a": "x", "b" : 1, "c" : [1, 2, 3]}, "product_description": "complex object" }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 7, "item_code" : [{"a": { "b" : 1}}, [1, 2, 3], 1, "x"], "product_description": "complex array" }', NULL);
SELECT documentdb_api.insert_one('db','catalog_items','{ "_id" : 8, "item_code" : [{"a": { "b" : 1}}, [1, 2, 3], 1, "x"], "product_description": "complex array" }', NULL);



-- Test filter generation 
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') FROM documentdb_api.collection('db', 'customer_purchases');

-- Test full lookup sql
WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match FROM documentdb_api.collection('db', 'customer_purchases')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE (array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Create Index on catalog_items
SELECT documentdb_api_internal.create_indexes_non_concurrently(
  'db',
  '{
     "createIndexes": "catalog_items",
     "indexes": [
       {"key": {"item_code": 1}, "name": "idx_catalog_items_item_code"}
     ]
   }',
   true
);

-- Test Index pushdown
SELECT documentdb_distributed_test_helpers.drop_primary_key('db','catalog_items');
BEGIN;
set local enable_seqscan TO off;
EXPLAIN(costs off) WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
FROM documentdb_api.collection('db', 'catalog_items') AS t2 
WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;
ROLLBACK;

-- Insert data into a new collection to be sharded (shard key can't be an array, but can be null or object)
SELECT documentdb_api.insert_one('db','customer_purchases_sharded',' { "_id" : 1, "item_name" : "shirt", "price" : 12, "order_quantity" : 2 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases_sharded','{ "_id" : 2, "item_name" : "pants", "price" : 20, "order_quantity" : 1 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases_sharded',' { "_id" : 3, "item_name" : "hat", "price" : 10, "order_quantity" : 5 }', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases_sharded',' { "_id" : 4}', NULL);
SELECT documentdb_api.insert_one('db','customer_purchases_sharded',' { "_id" : 5, "item_name" : {"a": "x", "b" : 1, "c" : [1, 2, 3]} }', NULL);

-- Shard customer_purchases collection on item_name 
SELECT documentdb_api.shard_collection('db','customer_purchases_sharded', '{"item_name":"hashed"}', false);

-- Test filter generation on sharded left collection
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') FROM documentdb_api.collection('db', 'customer_purchases_sharded');

-- Test Index pushdown on sharded left collection
BEGIN;
set local enable_seqscan TO off;
SET JIT To off;
SELECT documentdb_distributed_test_helpers.mask_plan_id_from_distributed_subplan($Q$
EXPLAIN(costs off) 
WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;
$Q$);
ROLLBACK;

SELECT shard_key_value, object_id, document FROM documentdb_api.collection('db', 'customer_purchases_sharded') order by object_id;


-- Test lookup on sharded left collection
WITH 
t1_0 AS(SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
  AS lookup_filters FROM documentdb_api.collection('db', 'customer_purchases_sharded')) 
SELECT bson_dollar_lookup_project(t1_0.document, t2_0_agg.matched_array , 'matched_docs'::text) as document 
FROM t1_0 LEFT JOIN LATERAL ( 
    WITH "LookupStage_0_0" as (SELECT * FROM documentdb_api.collection('db', 'catalog_items') AS t2_0 WHERE bson_dollar_in(t2_0.document, t1_0.lookup_filters) ) 
    SELECT COALESCE(array_agg("LookupStage_0_0".document::documentdb_core.bson), '{}'::bson[]) as matched_array FROM "LookupStage_0_0"
) t2_0_agg ON TRUE;

-- Test lookup on sharded left collection (rewrite avoiding the the CTE error)
WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;
 
-- Apply subpipeline on the out side of the join. This is a better execution plan since 
-- we are not applying the subpipeline on the entire 'from' collection
WITH simple_lookup as
(
WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded'))
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
-- perform lookup
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE
) 
-- subpipeline moved down 
SELECT bson_dollar_add_fields(bson_dollar_lookup_project, '{"matched_docs.hello": "newsubpipelinefield"}') from simple_lookup;


WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded')),
-- subpipeline moved up
t2 as (SELECT bson_dollar_add_fields(document, '{"matched_docs.hello": "newsubpipelinefield"}') as document from documentdb_api.collection('db', 'catalog_items'))
-- now perform lookup
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Lookup subpipeline applied after the join is done (this is the community version semantics)
WITH lookup_stage as
(
  -- Create CTE for the left/source collection (say t1)
  WITH                                                                                            
    t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	  AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded'))
  SELECT t1.document, t2_agg.document as matched
  FROM t1
  -- perform lateral join on the right collection (aka joined collection)
    LEFT JOIN LATERAL ( 
	  SELECT document
	  FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	  WHERE bson_dollar_in(t2.document, t1.match) 
  ) t2_agg ON TRUE
),
-- Apply subpipeline after lateral join 
subpipeline_stage1 as ( SELECT lookup_stage.document, bson_dollar_add_fields(lookup_stage.matched, '{"new": "field"}') as matched from lookup_stage)
-- Aggregate the results so, if there multiple matches they are folded in an array
SELECT bson_dollar_lookup_project(subpipeline_stage1.document, 
                COALESCE(array_agg(subpipeline_stage1.matched::documentdb_core.bson), '{}'::bson[]), 'matched_docs'::text) FROM subpipeline_stage1
GROUP BY subpipeline_stage1.document;

set citus.enable_local_execution to off;

-- Test lookup on sharded left collection
WITH 
t1_0 AS(SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
  AS lookup_filters FROM documentdb_api.collection('db', 'customer_purchases_sharded')) 
SELECT bson_dollar_lookup_project(t1_0.document, t2_0_agg.matched_array , 'matched_docs'::text) as document 
FROM t1_0 LEFT JOIN LATERAL ( 
    WITH "LookupStage_0_0" as (SELECT * FROM documentdb_api.collection('db', 'catalog_items') AS t2_0 WHERE bson_dollar_in(t2_0.document, t1_0.lookup_filters) ) 
    SELECT COALESCE(array_agg("LookupStage_0_0".document::documentdb_core.bson), '{}'::bson[]) as matched_array FROM "LookupStage_0_0"
) t2_0_agg ON TRUE;

-- Test lookup on sharded left collection (rewrite avoiding the the CTE error)
WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;
 
-- Apply subpipeline on the out side of the join. This is a better execution plan since 
-- we are not applying the subpipeline on the entire 'from' collection
WITH simple_lookup as
(
WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded'))
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
-- perform lookup
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE
) 
-- subpipeline moved down 
SELECT bson_dollar_add_fields(bson_dollar_lookup_project, '{"matched_docs.hello": "newsubpipelinefield"}') from simple_lookup;


WITH                                                                                            
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded')),
-- subpipeline moved up
t2 as (SELECT bson_dollar_add_fields(document, '{"matched_docs.hello": "newsubpipelinefield"}') as document from documentdb_api.collection('db', 'catalog_items'))
-- now perform lookup
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Lookup subpipeline applied after the join is done (this is the community version semantics)
WITH lookup_stage as
(
  -- Create CTE for the left/source collection (say t1)
  WITH                                                                                            
    t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"item_code" : "item_name"}') 
	  AS match, 'mdocs' asname FROM documentdb_api.collection('db', 'customer_purchases_sharded'))
  SELECT t1.document, t2_agg.document as matched
  FROM t1
  -- perform lateral join on the right collection (aka joined collection)
    LEFT JOIN LATERAL ( 
	  SELECT document
	  FROM documentdb_api.collection('db', 'catalog_items') AS t2 
	  WHERE bson_dollar_in(t2.document, t1.match) 
  ) t2_agg ON TRUE
),
-- Apply subpipeline after lateral join 
subpipeline_stage1 as ( SELECT lookup_stage.document, bson_dollar_add_fields(lookup_stage.matched, '{"new": "field"}') as matched from lookup_stage)
-- Aggregate the results so, if there multiple matches they are folded in an array
SELECT bson_dollar_lookup_project(subpipeline_stage1.document, 
                COALESCE(array_agg(subpipeline_stage1.matched::documentdb_core.bson), '{}'::bson[]), 'matched_docs'::text) FROM subpipeline_stage1
GROUP BY subpipeline_stage1.document;

set citus.enable_local_execution to on;

-- Test coalesce path, return empty array when no match found 
SELECT documentdb_api.insert_one('db','coalesce_source','{"_id": 0, "a": 1}', NULL);
SELECT documentdb_api.insert_one('db','coalesce_source','{"_id": 1, "a": null}', NULL);
SELECT documentdb_api.insert_one('db','coalesce_source','{"_id": 2}', NULL);
SELECT documentdb_api.insert_one('db','coalesce_foreign','{"_id": 0, "b": 1}', NULL);
SELECT documentdb_api.insert_one('db','coalesce_foreign','{"_id": 1, "b": null}', NULL);
SELECT documentdb_api.insert_one('db','coalesce_foreign','{"_id": 2}', NULL);

SELECT bson_dollar_lookup_extract_filter_expression(document, '{"nonex" : "a"}') FROM documentdb_api.collection('db', 'coalesce_source');
WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"nonex" : "a"}') 
	AS match FROM documentdb_api.collection('db', 'coalesce_source')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'coalesce_foreign') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Test dotted path

SELECT documentdb_api.insert_one('db','dotted_path_source','{"_id": 0, "a": 1}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_source','{"_id": 1, "a": null}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_source','{"_id": 2}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_source','{"_id": 3, "a": {"c": 1}}', NULL);

SELECT documentdb_api.insert_one('db','dotted_path_foreign','{"_id": 0, "b": 1}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_foreign','{"_id": 1, "b": null}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_foreign','{"_id": 2}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_foreign','{"_id": 3, "b": {"c" : 1}}', NULL);
SELECT documentdb_api.insert_one('db','dotted_path_foreign','{"_id": 4, "b": {"c" : 2}}', NULL);

SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b.c" : "a.c"}') FROM documentdb_api.collection('db', 'dotted_path_source');

WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"b.c" : "a.c"}') 
	AS match FROM documentdb_api.collection('db', 'dotted_path_source')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'dotted_path_foreign') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Empty from collection tests

SELECT * FROM documentdb_api.collection('db', 'from1NonExistent');

WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"b.c" : "a.c"}') 
	AS match FROM documentdb_api.collection('db', 'dotted_path_source')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'from1NonExistent') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Dotted path on mathched array name (some.documents)

SELECT documentdb_api.insert_one('db','dotted_as_source2','{"_id": 0, "a": {"b" : 1}}', NULL);
SELECT documentdb_api.insert_one('db','dotted_as_source2','{"_id": 1}', NULL);
SELECT documentdb_api.insert_one('db','dotted_as_foreign2','{"_id": 0, "target": 1}', NULL);

SELECT bson_dollar_lookup_extract_filter_expression(document, '{"target" : "a.b"}') FROM documentdb_api.collection('db', 'dotted_as_source2');
WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"target" : "a.b"}') 
	AS match FROM documentdb_api.collection('db', 'dotted_as_source2')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'same.documents'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'dotted_as_foreign2') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- Dotted path that goes through objects and arrays 

SELECT documentdb_api.insert_one('db','dotted_source','{"_id": 0, "a": [{"b": 1}, {"b": 2}]}', NULL);
SELECT documentdb_api.insert_one('db','dotted_foreign','{"_id": 0}', NULL);
SELECT documentdb_api.insert_one('db','dotted_foreign','{"_id": 1}', NULL);
SELECT documentdb_api.insert_one('db','dotted_foreign','{"_id": 2}', NULL);

SELECT bson_dollar_lookup_extract_filter_expression(document, '{"_id" : "a.b"}') FROM documentdb_api.collection('db', 'dotted_source');

-- Dotted AS path that is already present in the left document 
SELECT documentdb_api.insert_one('db','dotted_as_source','{"_id": 0, "a": [{"b": 1}, {"b": 2}]}', NULL);
SELECT documentdb_api.insert_one('db','dotted_as_source','{"_id": 1, "c": [{"d": 1}, {"d": 2}]}', NULL);

SELECT bson_dollar_lookup_extract_filter_expression(document, '{"c.d" : "a.b"}') FROM documentdb_api.collection('db', 'dotted_as_source');

WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"c.d" : "a.b"}') 
	AS match FROM documentdb_api.collection('db', 'dotted_as_source')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'a.b'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE(array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'dotted_as_source') AS t2 
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;

-- (B) Lookup with pipeline support

-- (B).1 Data Ingestion
SELECT documentdb_api.insert_one('db','establishments',' {"_id": 1, "establishment_name": "The Grand Diner", "dishes": ["burger", "fries"], "order_quantity": 100 , "drinks": ["soda", "juice"]}', NULL);
SELECT documentdb_api.insert_one('db','establishments','{ "_id": 2, "establishment_name": "Pizza Palace", "dishes": ["veggie pizza", "meat pizza"], "order_quantity": 120, "drinks": ["soda"]}', NULL);

SELECT documentdb_api.insert_one('db','menu','{ "_id": 1, "item_name": "burger", "establishment_name": "The Grand Diner"}', NULL);
SELECT documentdb_api.insert_one('db','menu','{ "_id": 2, "item_name": "veggie pizza", "establishment_name": "Pizza Palace", "drink": "water"}', NULL);
SELECT documentdb_api.insert_one('db','menu','{ "_id": 3, "item_name": "veggie pizza", "establishment_name": "Pizza Palace", "drink": "soda"}', NULL);

-- (B).1 Simple Lookup with pipeline

-- {
-- 	$lookup :
-- 	{
-- 		from : establishments,
--		to:	menu,
--		localField: establishment_name,
--		foreignField: establishment_name,
--		pipeline: {
--			[
--				{
--					$match :  { order_quantity : { $gt, 110}}
--				}
--			]
--		}
--	}
--}

SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "order_quantity": { "$gt": 110 } }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');


-- (B).2 Index creation on the from collection

  SELECT documentdb_api_internal.create_indexes_non_concurrently(
  'db',
  '{
     "createIndexes": "establishments",
     "indexes": [
       {"key": {"establishment_name": 1}, "name": "idx_establishments_establishment_name"},
	     {"key": {"order_quantity": 1}, "name": "idx_establishments_order_quantity"}
     ]
   }',
   true
);

-- (B).3.a Index usage
BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "order_quantity": { "$gt": 110 } }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

-- Adding a $sort in the pipeline
BEGIN;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "order_quantity": { "$gt": 110 } }}, { "$sort": { "_id": 1 }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

BEGIN;
SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "order_quantity": { "$gt": 110 } }}, { "$sort": { "_id": 1 }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

-- (B).3.b Index usage with optimization: (if lookup has a join condition and the lookup pipeline has $match as the first 
-- stage we push the $match filter up with the join. If both conditions are one same property both the filters should be 
-- part of the index condition)

BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "establishment_name" : { "$eq" : "Pizza Palace" } }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$match": { "establishment_name" : { "$eq" : "Pizza Palace" } }}], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

-- (B).4 Nested lookup pipeline (lookup pipeline containing lookup)

-- {
-- 	$lookup :
-- 	{
-- 		from : establishments,
--		to:	menu,
--		localField: establishment_name,
--		foreignField: establishment_name,
--		pipeline: {
--			[
--				{
--					$lookup :  {
-- 						from : establishments,
--						to:	establishments,
--						localField: _id,
--						foreignField: _id,
--						pipeline: {
--							[
--								{ unwind : "dishes"}
--							]
--						}
--					}
--				}
--			]
--		}
--	}
--}
SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "pipeline": [ { "$unwind": "$dishes" } ], "as": "matched_docs_id" } } ], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');

-- (B).5 Nested lookup pipeline (lookup pipeline containing lookup) index usage

BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "pipeline": [ { "$unwind": "$dishes" } ], "as": "matched_docs_id" } } ], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;

BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "as": "matched_docs_id" } } ], "cursor": {} }');
ROLLBACK;

BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "as": "matched_docs_id" } } ], "cursor": {} }');
ROLLBACK;


BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "pipeline": [ { "$project": { "bar": "$dishes" } } ], "as": "matched_docs_id" } } ], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;



BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "pipeline": [ { "$sort": { "dishes": 1 } } ], "as": "matched_docs_id" } } ], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;


BEGIN;
SET LOCAL enable_seqscan TO off;
EXPLAIN(costs off) SELECT document from bson_aggregation_pipeline('db', 
  '{ "aggregate": "menu", "pipeline": [ { "$lookup": { "from": "establishments", "pipeline": [ { "$lookup": { "from": "establishments", "localField": "_id", "foreignField": "_id", "pipeline": [ { "$project": { "_id": "bar" } } ], "as": "matched_docs_id" } } ], "as": "matched_docs", "localField": "establishment_name", "foreignField": "establishment_name" }} ], "cursor": {} }');
ROLLBACK;


-- Lookup Tests for array index-based paths

SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 1, "a" : [{"x" : 1}] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 2, "a" : {"x" : 1} }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 3, "a" : [{"x": 2}, {"x": 1}] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 4, "a" : [{"y": 1}, {"x": 1}] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 5, "a" : [2, 3] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 6, "a" : {"x": [1, 2]} }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 7, "a" : [{"x": 1}, {"y": 1}] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 8, "a" : [{"x": [1, 2]}, {"y": 1}] }', NULL);
SELECT documentdb_api.insert_one('db','array_path_purchases',' { "_id" : 9, "a" : [[{"x": [1, 2]}, {"y": 1}]] }', NULL);

SELECT documentdb_api.insert_one('db','array_path_items',' { "_id" : 1, "b" : 1 }', NULL);

-- Test filter generation 
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.x"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.0.x"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.x.0"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.0"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.0.1.y"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
SELECT bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.0.0.x"}'), document FROM documentdb_api.collection('db', 'array_path_purchases');
-- Test full lookup sql
WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"b" : "a.0.x"}') 
	AS match FROM documentdb_api.collection('db', 'array_path_purchases')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE (array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'array_path_items') AS t2
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;


WITH 
t1 AS (SELECT document, bson_dollar_lookup_extract_filter_expression(document, '{"a.0.x" : "b"}') 
	AS match FROM documentdb_api.collection('db', 'array_path_items')) 
SELECT bson_dollar_lookup_project(t1.document, t2_agg.agg, 'matched_docs'::text)
FROM t1 
LEFT JOIN LATERAL ( 
	SELECT COALESCE (array_agg(t2.document::documentdb_core.bson), '{}'::bson[]) as agg 
	FROM documentdb_api.collection('db', 'array_path_purchases') AS t2
	WHERE bson_dollar_in(t2.document, t1.match)
) t2_agg ON TRUE;


-- test lookup with operator type document contents
SELECT documentdb_api.insert_one('db', 'operator_lookup_left', '{ "_id": 1, "a": 1, "b": { "$isArray": 1, "OtherField": 1 }}');
SELECT documentdb_api.insert_one('db', 'operator_lookup_right', '{ "_id": 1, "a": 1, "b": { "$isArray": 1, "OtherField": 1 }}');

SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "operator_lookup_left", "pipeline": [ { "$lookup": { "from": "operator_lookup_right", "localField": "a", "foreignField": "a", "as": "myfoo" } }] }');

SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "operator_lookup_left", "pipeline": [ { "$lookup": { "from": "operator_lookup_right", "localField": "_id", "foreignField": "_id", "as": "myfoo" } }] }');
EXPLAIN (COSTS OFF) SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "operator_lookup_left", "pipeline": [ { "$lookup": { "from": "operator_lookup_right", "localField": "_id", "foreignField": "_id", "as": "myfoo" } }] }');

EXPLAIN (COSTS OFF) SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "operator_lookup_left", "pipeline": [ { "$project": { "_id": "500" }}, { "$lookup": { "from": "operator_lookup_right", "localField": "_id", "foreignField": "_id", "as": "myfoo" } }] }');

-- Test for Index push down crash when Foreign Field = '_id' and the target collection is sharded 
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 1, "item_name" : "itemA", "price" : 12, "order_quantity" : 2 }', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test','{ "_id" : 2, "item_name" : "itemE", "price" : 20, "order_quantity" : 1 }', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 3, "item_name" : "itemC", "price" : 10, "order_quantity" : 5 }', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 4, "item_name" : ["itemA", "itemC", "itemE"], "price" : 10, "order_quantity" : 5 }', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 5}', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 6, "item_name" : {"a": "x", "b" : 1, "c" : [1, 2, 3]} }', NULL);
SELECT documentdb_api.insert_one('db','purchases_pushdown_test',' { "_id" : 7, "item_name" : [{"a": { "b" : 1}}, [1, 2, 3], 1, "x"] }', NULL);

SELECT documentdb_api.insert_one('db','items_pushdown_test',' { "_id" :  "itemA", "item_description": "product 1", "in_stock" : 120 }', NULL);
SELECT documentdb_api.insert_one('db','items_pushdown_test',' { "_id" :  "itemB", "item_description": "product 1", "in_stock" : 240 }', NULL);
SELECT documentdb_api.insert_one('db','items_pushdown_test','{ "_id" :  "itemC", "item_description": "product 2", "in_stock" : 80 }', NULL);
SELECT documentdb_api.insert_one('db','items_pushdown_test','{ "_id" :  "itemD", "item_description": "product 3", "in_stock" : 60 }', NULL);
SELECT documentdb_api.insert_one('db','items_pushdown_test','{ "_id" :  "itemE", "item_description": "product 4", "in_stock" : 70 }', NULL);
SELECT documentdb_api.insert_one('db','items_pushdown_test','{ "_id" :  "itemF", "item_description": "product 4", "in_stock" : 70 }', NULL);

SELECT documentdb_api.shard_collection('db','items_pushdown_test', '{"_id":"hashed"}', false);


SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "purchases_pushdown_test", "pipeline": [ { "$lookup": { "from": "items_pushdown_test", "as": "matched_docs", "localField": "item_name", "foreignField": "_id", "pipeline": [ { "$count": "efe" } ] } } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id", "pipeline": [ { "$addFields": { "myBar": 1 } }, { "$limit": 10 }] } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs" } } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "includeArrayIndex": "idx" } } ], "cursor": {} }'); -- this will not inline

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "preserveNullAndEmptyArrays" : true } } ], "cursor": {} }'); -- should inline and use LEFT JOIN

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');
EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id", "pipeline": [ { "$addFields": { "myBar": 1 } }, { "$limit": 10 }] } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');
EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs" } } ], "cursor": {} }');
EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "includeArrayIndex": "idx" } } ], "cursor": {} }'); -- this will not inline
EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "preserveNullAndEmptyArrays" : true } } ], "cursor": {} }'); -- should inline and use LEFT JOIN



-- UDF Unit test for merge documents at path
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path(NULL, NULL, NULL);
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": "text"}', '{}', 'a');
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": "text"}', '{ "b" : true}', 'a');
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": "text"}', '{ "b" : true}', 'd');
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": { "b": "text", "c": true }}', '{ "random" : false }', 'a');
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": { "b": "text", "c": true }}', '{ "random" : false }', 'a.b');
SELECT documentdb_api_internal.bson_dollar_merge_documents_at_path('{"a": [{ "b": "text", "c": true }, { "b": "text2", "c": false }]}', '{ "random" : false }', 'a.b');

-- UDF tests for merge documents with override Array
SELECT documentdb_api_internal.bson_dollar_merge_documents(NULL, NULL, NULL);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": "text"}', '{}', false);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": "text"}', '{ "a" : true}', false);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": { "b": "text", "c": true }}', '{ "a.b.random" : false }', false);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": [{ "b": "text", "c": true }, { "b": "text2", "c": false }]}', '{ "a.b.random" : false }', false);

SELECT documentdb_api_internal.bson_dollar_merge_documents(NULL, NULL, NULL);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": "text"}', '{}', true);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": "text"}', '{ "a" : true}', true);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": { "b": "text", "c": true }}', '{ "a.b.random" : false }', true);
SELECT documentdb_api_internal.bson_dollar_merge_documents('{"a": [{ "b": "text", "c": true }, { "b": "text2", "c": false }]}', '{ "a.b.random" : false }', true);

-- todo: remove these duplicated tests
SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs" } } ], "cursor": {} }');

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "includeArrayIndex": "idx" } } ], "cursor": {} }'); -- this will not inline

SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "preserveNullAndEmptyArrays" : true } } ], "cursor": {} }'); -- should inline and use LEFT JOIN

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id", "pipeline": [ { "$addFields": { "myBar": 1 } }, { "$limit": 10 }] } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs" } } ], "cursor": {} }');

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "includeArrayIndex": "idx" } } ], "cursor": {} }'); -- this will not inline

EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "_id" } }, { "$unwind": { "path": "$matched_docs", "preserveNullAndEmptyArrays" : true } } ], "cursor": {} }'); -- should inline and use LEFT JOIN

-- Test for removal of inner join
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', '{ "createIndexes": "customer_purchases", "indexes": [ { "key": { "item_name": 1 }, "name": "customer_purchases_item_name_1" } ] }', TRUE);

BEGIN;
set local enable_seqscan to off;
SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$match": { "item_name": { "$exists": true } } }, { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "item_code" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');
ROLLBACK;

BEGIN;
set local enable_seqscan to off;
EXPLAIN (VERBOSE ON, COSTS OFF) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "customer_purchases", "pipeline": [ { "$match": { "item_name": { "$exists": true } } }, { "$lookup": { "from": "catalog_items", "as": "matched_docs", "localField": "item_name", "foreignField": "item_code" } }, { "$unwind": "$matched_docs" } ], "cursor": {} }');
ROLLBACK;

-- do a multi layer lookup that matches on _id
SELECT COUNT(documentdb_api.insert_one('db', 'source1', FORMAT('{ "_id": %s, "field1": "value%s", "fieldsource1": "foo" }', i, i)::bson)) FROM generate_series(1, 100) i;
SELECT COUNT(documentdb_api.insert_one('db', 'source2', FORMAT('{ "_id": "value%s", "field2": "othervalue%s", "fieldsource2": "foobar" }', i, i)::bson)) FROM generate_series(1, 100) i;
SELECT COUNT(documentdb_api.insert_one('db', 'source3', FORMAT('{ "_id": "othervalue%s", "field3": "someothervalue%s", "fieldsource3": "foobarfoo" }', i, i)::bson)) FROM generate_series(1, 100) i;
SELECT COUNT(documentdb_api.insert_one('db', 'source4', FORMAT('{ "_id": "someothervalue%s", "field4": "yetsomeothervalue%s", "fieldsource4": "foobarbaz" }', i, i)::bson)) FROM generate_series(1, 100) i;

-- create indexes on the intermediate fields
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', '{ "createIndexes": "source1", "indexes": [ { "key": { "field1": 1 }, "name": "field1_1" } ] }', TRUE);

ANALYZE documentdb_data.documents_6020;
ANALYZE documentdb_data.documents_6021;
ANALYZE documentdb_data.documents_6022;
ANALYZE documentdb_data.documents_6023;

-- should always pick up _id index.
BEGIN;
set local enable_indexscan to off;
set local enable_bitmapscan to off;
set local enable_material to off;
set local documentdb.useLegacyForcePushdownBehavior to off;
EXPLAIN (COSTS OFF, ANALYZE ON, VERBOSE ON, TIMING OFF, SUMMARY OFF) SELECT document FROM bson_aggregation_pipeline('db',
  '{ "aggregate": "source1", "pipeline": [ 
    { "$match": { "field1": { "$eq": "value10" } } },
    { "$lookup": { "from": "source2", "as": "source2_docs", "localField": "field1", "foreignField": "_id" } },
    { "$unwind": { "path": "$source2_docs", "preserveNullAndEmptyArrays": true } },
    { "$lookup": { "from": "source3", "as": "source3_docs", "localField": "source2_docs", "foreignField": "_id" } },
    { "$unwind": { "path": "$source3_docs", "preserveNullAndEmptyArrays": true } },
    { "$lookup": { "from": "source4", "as": "source4_docs", "localField": "source3_docs", "foreignField": "_id" } },
    { "$unwind": { "path": "$source4_docs", "preserveNullAndEmptyArrays": true } }
  ]}');
ROLLBACK;

-- Test lookup query building works with aggregates
EXPLAIN (COSTS OFF, VERBOSE ON) SELECT document FROM bson_aggregation_pipeline('db', 
    '{ "aggregate": "abc", "pipeline": [
      { "$lookup": 
       { 
        "from": "def", 
        "as": "lookup_count", 
        "let" : { "a" : "$A", "b" : "$B", "c" : "$C" }, 
        "pipeline": [
         { 
          "$group": { 
           "_id": "$groupId",
           "total_score": { "$sum": "$Abc.score" }, 
           "count": { "$sum": 1 } 
          } 
        }] 
      }}]}');