/*
Copyright 2013 Red Hat, Inc. and/or its affiliates.
This file is part of lightblue.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.redhat.lightblue.assoc;
import java.util.Set;
import java.util.HashSet;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.Executors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.redhat.lightblue.query.QueryExpression;
import com.redhat.lightblue.crud.CRUDFindResponse;
import com.redhat.lightblue.crud.CRUDFindRequest;
import com.redhat.lightblue.crud.DocCtx;
import com.redhat.lightblue.crud.ListDocumentStream;
import com.redhat.lightblue.metadata.CompositeMetadata;
import com.redhat.lightblue.mediator.Finder;
import com.redhat.lightblue.mediator.OperationContext;
import com.redhat.lightblue.mediator.Mediator;
import com.redhat.lightblue.assoc.scorers.IndexedFieldScorer;
import com.redhat.lightblue.assoc.iterators.BruteForceQueryPlanIterator;
import com.redhat.lightblue.assoc.scorers.SimpleScorer;
import com.redhat.lightblue.assoc.iterators.First;
import com.redhat.lightblue.assoc.ep.ExecutionPlan;
import com.redhat.lightblue.assoc.ep.StepResult;
import com.redhat.lightblue.assoc.ep.ResultDocument;
import com.redhat.lightblue.assoc.ep.ExecutionContext;
import com.redhat.lightblue.assoc.ep.MakeDocCtx;
import com.redhat.lightblue.assoc.ep.StepResultDocumentStream;
import com.redhat.lightblue.util.JsonDoc;
/**
* Finder for searches involving composite entities
*
* This implementation builds a search and retrieval plan based on the query
* plan. The generation of search plan is optional, if it discovers that the
* entities can be searched and retrieve with a single plan, it only generates a
* retrieval plan. The retrieval plan has the same structure as the entity
* composite metadata. First the documents for the root entity are retrieved,
* then, recursively, the associated documents.
*
* The search plan, if exists, has a different structure than the document
* structure. It is based on the request query, and terminates as soon as the
* root entity documents are retrieved. Once the root documents are retrieved,
* that list is used to execute the retrieval plan.
*/
public class CompositeFindImpl implements Finder {
private static final Logger LOGGER = LoggerFactory.getLogger(CompositeFindImpl.class);
private final CompositeMetadata root;
// Instance state data. This class is meant to be thrown away after used once.
// Query analysis information for the request query
private transient List<QueryFieldInfo> requestQueryInfo;
// The search and retrieval query plans
// If searchQPlan is null, then retrievalQPlan performs both search and retrieval
// If searchQPlan is not null, then searchQPlan searches the documents, and retrievalQPlan
// retrieves the found documents
private transient QueryPlan searchQPlan;
private transient QueryPlan retrievalQPlan;
private transient ExecutionPlan executionPlan;
private int parallelism = 1;
public CompositeFindImpl(CompositeMetadata md) {
this.root = md;
}
/**
* Set maximum number of threads that can run in parallel. There's a hard
* limit on 10
*/
public void setParallelism(int n) {
parallelism = n;
if (parallelism < 1) {
parallelism = 1;
}
if (parallelism > 10) {
parallelism = 10;
}
}
private void initialize(OperationContext ctx,
CRUDFindRequest req) {
if(executionPlan==null) {
// Composite find algorithm works like this:
// 1) Find the minimal entity tree required to evaluate the request.query
// 2) Find a query plan for that minimal entity tree
// 3) If the query plan root is also the root entity, then expand that query
// plan to include all nodes
// 4) Execute search based on query plan
// 5) If query plan node is not the entity root, find documents found for entity root,
// and re-retrieve the documents
// First: detemine minimal entity tree containing the nodes sufficient to
// evaluate the query
Set<CompositeMetadata> minimalTree = findMinimalSetOfQueryEntities(req.getQuery(),
ctx.getTopLevelEntityMetadata());
selectQueryPlan(req.getQuery(), minimalTree);
LOGGER.debug("Search query plan:{}, retrieval query plan:{}", searchQPlan, retrievalQPlan);
executionPlan = new ExecutionPlan(req.getQuery(),
req.getProjection(),
req.getSort(),
req.getFrom(),
req.getTo(),
root,
searchQPlan,
retrievalQPlan);
}
}
@Override
public void explain(OperationContext ctx,
CRUDFindRequest req) {
initialize(ctx,req);
ExecutionContext executionContext = new ExecutionContext(ctx,null);
JsonDoc doc=new JsonDoc(executionPlan.explain(executionContext));
ctx.setDocumentStream(new ListDocumentStream<DocCtx>(Arrays.asList(new DocCtx(doc))));
}
@Override
public CRUDFindResponse find(OperationContext ctx,
CRUDFindRequest req) {
LOGGER.debug("Composite find: start");
initialize(ctx,req);
ctx.setProperty(Mediator.CTX_QPLAN, searchQPlan == null ? retrievalQPlan : searchQPlan);
LOGGER.debug("Execution plan:{}", executionPlan);
CRUDFindResponse response = new CRUDFindResponse();
ExecutionContext executionContext = new ExecutionContext(ctx,
Executors.newWorkStealingPool(parallelism));
try {
StepResult<ResultDocument> results = executionPlan.getResults(executionContext);
ctx.setDocumentStream(new StepResultDocumentStream(new MakeDocCtx(results)));
response.setSize(executionContext.getMatchCount());
LOGGER.debug("Composite find: end");
return response;
} finally {
executionContext.close();
}
}
/**
* Selects the search and retrieval query plans based on the minimal tree
* and request query.
*
* There is either only a retrieval plan, or both a search plan and
* retrieval plan. If search plan cannot retrieve the root entity, then we
* use the search plan to collect the entities matching the search criteria,
* and then the retrieval plan to retrieve those entities completely. If the
* search plan can both search and retrieve the entities, there will be only
* a retrieval plan.
*
*
*/
private void selectQueryPlan(QueryExpression requestQuery,
Set<CompositeMetadata> minimalTree) {
searchQPlan = retrievalQPlan = null;
if (minimalTree.size() > 1) {
// There are multiple entities required to evaluate the query
// Choose a query plan
QueryPlan searchQP = new QueryPlanChooser(root,
new BruteForceQueryPlanIterator(),
new IndexedFieldScorer(),
requestQuery,
minimalTree).choose();
LOGGER.debug("Candidate plan: {}", searchQP);
// If the query plan has only one source, and that source is the root, then
// we don't need to search and retrieve in two separate steps, we can simply
// retrieve everything while we search
QueryPlanNode[] roots = searchQP.getSources();
if (roots.length == 1 && roots[0].getMetadata() == root) {
LOGGER.debug("Search is trivial, root node is at query plan root, so search and retrieve");
// Build a new query plan containing all entities. This plan should
// have the same root as before. If not, something must be
// wrong, and we fall back to a search/retrieve query
QueryPlan fullPlan = new QueryPlanChooser(root,
new BruteForceQueryPlanIterator(),
new IndexedFieldScorer(),
requestQuery,
null).choose();
// This plan must also have a single root
roots = fullPlan.getSources();
if (roots.length == 1 && roots[0].getMetadata() == root) {
// Retrieve everything, no separate search plan
retrievalQPlan = fullPlan;
searchQPlan = null;
} else {
// Search and retrieve in separate phases
searchQPlan = searchQP;
}
} else {
// Multiple roots, search and retrieve in separate phases
searchQPlan = searchQP;
}
} else {
// Minimal tree has only one entity. That entity must be the root entity
// That means, a single retrieval plan can search and retrieve
searchQPlan = null;
}
if (retrievalQPlan == null) {
if (searchQPlan == null) {
// Search and retrieve
retrievalQPlan = new QueryPlanChooser(root,
new First(),
new SimpleScorer(),
requestQuery,
null).choose();
} else {
// No search, only retrieve. No query.
retrievalQPlan = new QueryPlanChooser(root,
new First(),
new SimpleScorer(),
null,
null).choose();
}
}
}
/**
* Determine which entities are required to evaluate the given query
*/
private Set<CompositeMetadata> findMinimalSetOfQueryEntities(QueryExpression query,
CompositeMetadata md) {
Set<CompositeMetadata> entities = new HashSet<>();
if (query != null) {
AnalyzeQuery aq = new AnalyzeQuery(md, null);
aq.iterate(query);
requestQueryInfo = aq.getFieldInfo();
LOGGER.debug("Analyze query results for {}: {}", query, requestQueryInfo);
for (QueryFieldInfo fi : requestQueryInfo) {
CompositeMetadata e = fi.getFieldEntity();
if (e != md) {
entities.add(e);
}
}
// All entities on the path from every entity to the root
// should also be included
Set<CompositeMetadata> intermediates = new HashSet<>();
for (CompositeMetadata x : entities) {
CompositeMetadata trc = x.getParent();
while (trc != null) {
intermediates.add(trc);
trc = trc.getParent();
}
}
entities.addAll(intermediates);
}
// At this point, entities contains all required entities, but maybe not the root
entities.add(md);
LOGGER.debug("Minimal set of entities required to evaluate {}:{}", query, entities);
return entities;
}
}