From 7556fa8f47218cd79d64f9304c35281a7f77dd0c Mon Sep 17 00:00:00 2001 From: lling Date: Thu, 15 Oct 2020 11:01:56 -0700 Subject: [PATCH 1/2] Add CtsQueryBuilder for DMSDK in Java API #1260 --- .../datamovement/DataMovementManager.java | 16 +++-- .../impl/DataMovementManagerImpl.java | 14 ++-- .../impl/DataMovementServices.java | 4 +- .../datamovement/impl/QueryBatcherImpl.java | 20 +++--- .../client/expression/CtsQueryBuilder.java | 68 +++++++++++++++++++ .../marklogic/client/impl/OkHttpServices.java | 31 ++++++--- .../client/impl/QueryManagerImpl.java | 2 +- .../marklogic/client/impl/RESTServices.java | 4 +- .../test/datamovement/QueryBatcherTest.java | 30 +++++--- 9 files changed, 142 insertions(+), 47 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java index f05f5f205..f33b809e9 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java @@ -17,11 +17,7 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.client.io.marker.ContentHandle; -import com.marklogic.client.query.RawCtsQueryDefinition; -import com.marklogic.client.query.StringQueryDefinition; -import com.marklogic.client.query.StructuredQueryDefinition; -import com.marklogic.client.query.RawCombinedQueryDefinition; -import com.marklogic.client.query.RawStructuredQueryDefinition; +import com.marklogic.client.query.*; import java.util.Iterator; @@ -127,6 +123,16 @@ public interface DataMovementManager { */ public WriteBatcher newWriteBatcher(); + /** + * Create a new QueryBatcher instance configured to retrieve uris that + * match this query. + * + * @param query the query used to find matching uris + * + * @return the new QueryBatcher instance + */ + public QueryBatcher newQueryBatcher(CtsQueryDefinition query); + /** * Create a new QueryBatcher instance configured to retrieve uris that * match this query. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java index ff6953f65..598a87f1d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java @@ -20,12 +20,7 @@ import com.marklogic.client.datamovement.*; import com.marklogic.client.impl.DatabaseClientImpl; import com.marklogic.client.io.marker.ContentHandle; -import com.marklogic.client.query.QueryDefinition; -import com.marklogic.client.query.RawCtsQueryDefinition; -import com.marklogic.client.query.StringQueryDefinition; -import com.marklogic.client.query.StructuredQueryDefinition; -import com.marklogic.client.query.RawCombinedQueryDefinition; -import com.marklogic.client.query.RawStructuredQueryDefinition; +import com.marklogic.client.query.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,6 +97,11 @@ public WriteBatcher newWriteBatcher() { return batcher; } + @Override + public QueryBatcher newQueryBatcher(CtsQueryDefinition query) { + return newQueryBatcherImpl(query); + } + @Override public QueryBatcher newQueryBatcher(StructuredQueryDefinition query) { return newQueryBatcherImpl(query); @@ -126,7 +126,7 @@ public QueryBatcher newQueryBatcher(RawCtsQueryDefinition query) { return newQueryBatcherImpl(query); } - private QueryBatcher newQueryBatcherImpl(QueryDefinition query) { + private QueryBatcher newQueryBatcherImpl(SearchQueryDefinition query) { if ( query == null ) throw new IllegalArgumentException("query must not be null"); QueryBatcherImpl queryBatcher = null; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java index e42fd3e22..1a65fc028 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java @@ -27,7 +27,7 @@ import com.marklogic.client.impl.DatabaseClientImpl; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.datamovement.JobTicket.JobType; -import com.marklogic.client.query.QueryDefinition; +import com.marklogic.client.query.SearchQueryDefinition; import com.marklogic.client.util.RequestParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +48,7 @@ public DataMovementServices setClient(DatabaseClient client) { return this; } - QueryConfig initConfig(String method, QueryDefinition qdef) { + QueryConfig initConfig(String method, SearchQueryDefinition qdef) { logger.debug("initializing forest configuration with query"); if (qdef == null) throw new IllegalArgumentException("null query definition"); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java index 248876145..ef6e39f99 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java @@ -33,12 +33,12 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.io.marker.StructureWriteHandle; import com.marklogic.client.query.RawQueryDefinition; +import com.marklogic.client.query.SearchQueryDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.marklogic.client.DatabaseClient; import com.marklogic.client.ResourceNotFoundException; -import com.marklogic.client.query.QueryDefinition; import com.marklogic.client.impl.QueryManagerImpl; import com.marklogic.client.impl.UrisHandle; @@ -59,8 +59,8 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private static Logger logger = LoggerFactory.getLogger(QueryBatcherImpl.class); private String queryMethod; - private QueryDefinition query; - private QueryDefinition originalQuery; + private SearchQueryDefinition query; + private SearchQueryDefinition originalQuery; private Boolean filtered; private Iterator iterator; private boolean threadCountSet = false; @@ -84,7 +84,7 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private long maxBatches = Long.MAX_VALUE; QueryBatcherImpl( - QueryDefinition originalQuery, DataMovementManager moveMgr, ForestConfiguration forestConfig, + SearchQueryDefinition originalQuery, DataMovementManager moveMgr, ForestConfiguration forestConfig, String serializedCtsQuery, Boolean filtered ) { this(moveMgr, forestConfig); @@ -100,7 +100,7 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { initQuery(originalQuery); } } - public QueryBatcherImpl(QueryDefinition query, DataMovementManager moveMgr, ForestConfiguration forestConfig) { + public QueryBatcherImpl(SearchQueryDefinition query, DataMovementManager moveMgr, ForestConfiguration forestConfig) { this(moveMgr, forestConfig); initQuery(query); } @@ -113,7 +113,7 @@ private QueryBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forest withForestConfig(forestConfig); withBatchSize(1000); } - private void initQuery(QueryDefinition query) { + private void initQuery(SearchQueryDefinition query) { if (query == null) { throw new IllegalArgumentException("Cannot create QueryBatcher with null query"); } @@ -589,7 +589,7 @@ private class QueryTask implements Runnable { private QueryBatcherImpl batcher; private Forest forest; private String queryMethod; - private QueryDefinition query; + private SearchQueryDefinition query; private Boolean filtered; private long forestBatchNum; private long start; @@ -599,17 +599,17 @@ private class QueryTask implements Runnable { private String nextAfterUri; QueryTask(DataMovementManager moveMgr, QueryBatcherImpl batcher, Forest forest, - String queryMethod, QueryDefinition query, Boolean filtered, long forestBatchNum, long start + String queryMethod, SearchQueryDefinition query, Boolean filtered, long forestBatchNum, long start ) { this(moveMgr, batcher, forest, queryMethod, query, filtered, forestBatchNum, start, null, -1, true); } QueryTask(DataMovementManager moveMgr, QueryBatcherImpl batcher, Forest forest, - String queryMethod, QueryDefinition query, Boolean filtered, long forestBatchNum, long start, String afterUri + String queryMethod, SearchQueryDefinition query, Boolean filtered, long forestBatchNum, long start, String afterUri ) { this(moveMgr, batcher, forest, queryMethod, query, filtered, forestBatchNum, start, afterUri, -1, true); } QueryTask(DataMovementManager moveMgr, QueryBatcherImpl batcher, Forest forest, - String queryMethod, QueryDefinition query, Boolean filtered, long forestBatchNum, long start, String afterUri, + String queryMethod, SearchQueryDefinition query, Boolean filtered, long forestBatchNum, long start, String afterUri, long retryBatchNumber, boolean callFailListeners ) { this.moveMgr = moveMgr; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/CtsQueryBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/CtsQueryBuilder.java index 974e7730c..93db0d243 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/CtsQueryBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/CtsQueryBuilder.java @@ -20,19 +20,69 @@ import com.marklogic.client.query.CtsQueryDefinition; import com.marklogic.client.type.CtsQueryExpr; +/** + * CtsQueryBuilder builds a query for documents in the database. + */ public abstract class CtsQueryBuilder { + /** + * Builds expressions with cts server functions. + */ public final CtsExpr cts; + + /** + * Builds expressions with fn server functions. + */ public final FnExpr fn; + + /** + * Builds expressions with geo server functions. + */ public final GeoExpr geo; + + /** + * Builds expressions with json server functions. + */ public final JsonExpr json; + + /** + * Builds expressions with map server functions. + */ public final MapExpr map; + + /** + * Builds expressions with math server functions. + */ public final MathExpr math; + + /** + * Builds expressions with rdf server functions. + */ public final RdfExpr rdf; + + /** + * Builds expressions with sem server functions. + */ public final SemExpr sem; + + /** + * Builds expressions with spell server functions. + */ public final SpellExpr spell; + + /** + * Builds expressions with sql server functions. + */ public final SqlExpr sql; + + /** + * Builds expressions with xdmp server functions. + */ public final XdmpExpr xdmp; + + /** + * Builds expressions with xs server functions. + */ public final XsExpr xs; protected CtsQueryBuilder( @@ -54,9 +104,27 @@ protected CtsQueryBuilder( } + /** + * Create a CtsQueryDefinition based on a cts query + * @param query a cts query + * @return a CtsQueryDefinition + */ public abstract CtsQueryDefinition newCtsQueryDefinition(CtsQueryExpr query); + /** + * Create a CtsQueryDefinition based on a cts query and query options + * @param query a cts query + * @param queryOptions query options + * @return a CtsQueryDefinition + */ public abstract CtsQueryDefinition newCtsQueryDefinition(CtsQueryExpr query, JSONWriteHandle queryOptions); + /** + * Export cts query into a handle in AST format + * @param query the cts query to be exported + * @param handle the handle to store exported query + * @param the handle type + * @return a handle which contains exported cts query in AST format + */ public abstract T export(CtsQueryExpr query, T handle); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index b7ca676a5..32e1d5a4d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -3010,7 +3010,7 @@ public R getSystemSchema(RequestLogger reqlog, St return getResource(reqlog, "internal/schemas", null, params, output); } @Override - public R uris(RequestLogger reqlog, String method, QueryDefinition qdef, + public R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { logger.debug("Querying for uris"); @@ -3024,19 +3024,25 @@ public R uris(RequestLogger reqlog, String method, Qu } @Override public R forestInfo(RequestLogger reqlog, - String method, RequestParameters params, QueryDefinition qdef, R output + String method, RequestParameters params, SearchQueryDefinition qdef, R output ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { return processQuery(reqlog, "internal/forestinfo", method, params, qdef, output); } private R processQuery(RequestLogger reqlog, String path, - String method, RequestParameters params, QueryDefinition qdef, R output + String method, RequestParameters params, SearchQueryDefinition qdef, R output ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException { - if (qdef.getDirectory() != null) params.add("directory", qdef.getDirectory()); - if (qdef.getCollections() != null ) { - for ( String collection : qdef.getCollections() ) { - params.add("collection", collection); + if (qdef instanceof QueryDefinition) { + if (((QueryDefinition)qdef).getDirectory() != null) { + params.add("directory", ((QueryDefinition)qdef).getDirectory()); + } + + if (((QueryDefinition)qdef).getCollections() != null ) { + for ( String collection : ((QueryDefinition)qdef).getCollections() ) { + params.add("collection", collection); + } + } } - } + if (qdef.getOptionsName()!= null && qdef.getOptionsName().length() > 0) { params.add("options", qdef.getOptionsName()); } @@ -3098,7 +3104,14 @@ private R processQuery(RequestLogger reqlog, Stri RawQueryDefinition rawQuery = (RawQueryDefinition) qdef; logger.debug("{} processing raw query", path); input = checkFormat(rawQuery.getHandle()); - } else { + } else if (qdef instanceof CtsQueryDefinition) { + CtsQueryDefinition builtCtsQuery = (CtsQueryDefinition) qdef; + structure = builtCtsQuery.serialize(); + logger.debug("{} processing cts query {}", path, structure); + if (sendQueryAsPayload && structure != null) { + input = new StringHandle(structure).withFormat(Format.TEXT); + } + }else { throw new UnsupportedOperationException(path+" cannot process query of "+qdef.getClass().getName()); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/QueryManagerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/QueryManagerImpl.java index 264c00d3b..e57407860 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/QueryManagerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/QueryManagerImpl.java @@ -164,7 +164,7 @@ public T search(SearchQueryDefinition querydef, T s return services.search(requestLogger, searchHandle, querydef, start, pageLen, view, transaction, forestName); } - public T uris(String method, QueryDefinition querydef, Boolean filtered, T urisHandle, + public T uris(String method, SearchQueryDefinition querydef, Boolean filtered, T urisHandle, long start, String afterUri, String forestName) { return services.uris(requestLogger, method, querydef, filtered, start, afterUri, pageLen, forestName, urisHandle); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java index 5d23f9fbd..7e9db5499 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RESTServices.java @@ -219,11 +219,11 @@ void deleteValues(RequestLogger logger, String type) R getSystemSchema(RequestLogger reqlog, String schemaName, R output) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; - R uris(RequestLogger reqlog, String method, QueryDefinition qdef, + R uris(RequestLogger reqlog, String method, SearchQueryDefinition qdef, Boolean filtered, long start, String afterUri, long pageLength, String forestName, R output) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; R forestInfo(RequestLogger reqlog, - String method, RequestParameters params, QueryDefinition qdef, R output + String method, RequestParameters params, SearchQueryDefinition qdef, R output ) throws ResourceNotFoundException, ForbiddenUserException, FailedRequestException; R getResource(RequestLogger reqlog, String path, diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/QueryBatcherTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/QueryBatcherTest.java index 861a70b9e..d9448d6e1 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/QueryBatcherTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/QueryBatcherTest.java @@ -44,9 +44,11 @@ import com.marklogic.client.FailedRequestException; import com.marklogic.client.datamovement.impl.DataMovementManagerImpl; +import com.marklogic.client.expression.CtsQueryBuilder; import com.marklogic.client.io.Format; -import com.marklogic.client.query.RawCtsQueryDefinition; +import com.marklogic.client.query.*; +import com.marklogic.client.type.CtsQueryExpr; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; @@ -72,14 +74,7 @@ import com.marklogic.client.io.StringHandle; import static com.marklogic.client.io.Format.JSON; import static com.marklogic.client.io.Format.XML; -import com.marklogic.client.query.DeleteQueryDefinition; -import com.marklogic.client.query.QueryDefinition; -import com.marklogic.client.query.RawCombinedQueryDefinition; -import com.marklogic.client.query.RawStructuredQueryDefinition; -import com.marklogic.client.query.StructuredQueryDefinition; -import com.marklogic.client.query.StringQueryDefinition; -import com.marklogic.client.query.QueryManager; -import com.marklogic.client.query.StructuredQueryBuilder; + import com.marklogic.client.datamovement.ApplyTransformListener; import com.marklogic.client.datamovement.DataMovementManager; import com.marklogic.client.datamovement.DeleteListener; @@ -252,6 +247,19 @@ public void testRawValueQuery() throws Exception { runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 17, 99); } + @Test + public void testCtsQuery() throws Exception { + CtsQueryBuilder ctsQueryBuilder = client.newQueryManager().newCtsSearchBuilder(); + CtsQueryExpr ctsQueryExpr = ctsQueryBuilder.cts.wordQuery("Doe"); + CtsQueryDefinition query = ctsQueryBuilder.newCtsQueryDefinition(ctsQueryExpr); + query.setOptionsName("employees"); + Map matchesByForest = new HashMap<>(); + matchesByForest.put("java-unittest-1", new String[] {uri1}); + matchesByForest.put("java-unittest-2", new String[] {}); + matchesByForest.put("java-unittest-3", new String[] {uri2}); + runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 99, 17); + } + @Test public void testIterator() throws Exception { Map matchesByForest = new HashMap<>(); @@ -291,7 +299,7 @@ public void testRawCombinedQuery() throws Exception { runQueryBatcher(moveMgr.newQueryBatcher(query), query, matchesByForest, 30, 20); } - public void runQueryBatcher(QueryBatcher queryBatcher, QueryDefinition query, Map matchesByForest, + public void runQueryBatcher(QueryBatcher queryBatcher, SearchQueryDefinition query, Map matchesByForest, int batchSize, int threadCount, boolean queryBatcherChecks) throws Exception { String queryBatcherJobId = "QueryBatcherJobId"; String queryBatcherJobName = "QueryBatcherJobName"; @@ -419,7 +427,7 @@ public void runQueryBatcher(QueryBatcher queryBatcher, QueryDefinition query, Ma } } - public void runQueryBatcher(QueryBatcher queryBatcher, QueryDefinition query, Map matchesByForest, + public void runQueryBatcher(QueryBatcher queryBatcher, SearchQueryDefinition query, Map matchesByForest, int batchSize, int threadCount) throws Exception { runQueryBatcher(queryBatcher, query, matchesByForest, batchSize, threadCount, true); } From 4a186dffee755331c71953575d673fbc7df552eb Mon Sep 17 00:00:00 2001 From: lling Date: Thu, 15 Oct 2020 11:47:05 -0700 Subject: [PATCH 2/2] Change format from TEXT to JSON #1260 --- .../src/main/java/com/marklogic/client/impl/OkHttpServices.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 32e1d5a4d..bd20593f3 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -3109,7 +3109,7 @@ private R processQuery(RequestLogger reqlog, Stri structure = builtCtsQuery.serialize(); logger.debug("{} processing cts query {}", path, structure); if (sendQueryAsPayload && structure != null) { - input = new StringHandle(structure).withFormat(Format.TEXT); + input = new StringHandle(structure).withFormat(Format.JSON); } }else { throw new UnsupportedOperationException(path+" cannot process query of "+qdef.getClass().getName());