diff --git a/contrib/datawave-quickstart/bin/services/datawave/test-web/tests/ModelBean.test b/contrib/datawave-quickstart/bin/services/datawave/test-web/tests/ModelBean.test new file mode 100644 index 00000000000..dd12723a342 --- /dev/null +++ b/contrib/datawave-quickstart/bin/services/datawave/test-web/tests/ModelBean.test @@ -0,0 +1,102 @@ +modelName=Model1 +modelFile=${DW_DATAWAVE_SOURCE_DIR}/web-services/model/src/test/resources/ModelBeanTest_m1.xml + +################################################################ +# Test /Model/import endpoint +# + +setCurlData @${modelFile} + +configureTest \ + ImportQueryModel200 \ + "Imports an example query model named '${modelName}'" \ + "--header 'Content-Type: application/xml;charset=UTF-8' ${DW_CURL_DATA} -X POST ${URI_ROOT}/Model/import" \ + "application/xml;charset=UTF-8" \ + 200 + +runTest + +################################################################ +# Test GET /Model/admin/{name} endpoint (200 b/c cache bypassed) +# + +configureTest \ + AdminGetQueryModel200 \ + "Retrieve the imported query model" \ + "-X GET ${URI_ROOT}/Model/admin/${modelName}" \ + "application/xml;charset=UTF-8" \ + 200 + +runTest + +################################################################ +# Retry /Model/import on the same file +# + +setCurlData @${modelFile} + +configureTest \ + ImportQueryModelAgain412 \ + "Try to import the same query model, expecting 412 response b/c model exists" \ + "--header 'Content-Type: application/xml;charset=UTF-8' ${DW_CURL_DATA} -X POST ${URI_ROOT}/Model/import" \ + "application/xml;charset=UTF-8" \ + 412 + +runTest + +################################################################ +# Retry import on the same file, but this time using -X PUT +# '/Model/import/{modelName}' to enable update/replace +# + +setCurlData @${modelFile} + +configureTest \ + ImportQueryModelAgainWithPUT200 \ + "Try to import the same query model, expecting 200 response b/c '/Model/import/{modelName}' was used" \ + "--header 'Content-Type: application/xml;charset=UTF-8' ${DW_CURL_DATA} -X PUT ${URI_ROOT}/Model/import/${modelName}" \ + "application/xml;charset=UTF-8" \ + 200 + +runTest + +################################################################ +# Attempt '/Model/import/WrongModelName' and get 412 error +# + +setCurlData @${modelFile} + +configureTest \ + ImportQueryModelAgainWithPUT412 \ + "Try to update/replace but pass incorrect model name, expecting 412" \ + "--header 'Content-Type: application/xml;charset=UTF-8' ${DW_CURL_DATA} -X PUT ${URI_ROOT}/Model/import/WrongModelName" \ + "application/xml;charset=UTF-8" \ + 412 + +runTest + +################################################################ +# Test DELETE /Model/{name} endpoint +# + +configureTest \ + DeleteQueryModel200 \ + "Delete the imported query model" \ + "-X DELETE ${URI_ROOT}/Model/${modelName}" \ + "application/xml;charset=UTF-8" \ + 200 + +runTest + +################################################################ +# Try to GET the deleted model with admin endpoint: 404 b/c cache is bypassed +# + +configureTest \ + GetDeletedQueryModelAndFail404 \ + "Try and fail to retrieve the deleted query model, as appropriate" \ + "-X GET ${URI_ROOT}/Model/admin/${modelName}" \ + "application/xml;charset=UTF-8" \ + 404 + +# This last test is executed by run.sh, as usual diff --git a/pom.xml b/pom.xml index adf28228acb..9e1fc3676af 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 2.5.2 1.6.0 3.0.0 - 3.0.2 + 3.0.3 3.0.0 3.0.0 3.0.0 diff --git a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java index c597d0e79fd..b0fbbab08f3 100644 --- a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java +++ b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java @@ -23,9 +23,11 @@ import javax.interceptor.Interceptors; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; @@ -37,6 +39,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; @@ -58,6 +61,8 @@ import datawave.security.util.ScannerHelper; import datawave.webservice.common.cache.AccumuloTableCache; import datawave.webservice.common.connection.AccumuloConnectionFactory; +import datawave.webservice.common.connection.WrappedAccumuloClient; +import datawave.webservice.common.connection.WrappedScannerHelper; import datawave.webservice.common.exception.DatawaveWebApplicationException; import datawave.webservice.common.exception.NotFoundException; import datawave.webservice.common.exception.PreConditionFailedException; @@ -86,7 +91,7 @@ public class ModelBean { private static final long BATCH_WRITER_MAX_MEMORY = 10845760; private static final int BATCH_WRITER_MAX_THREADS = 2; - private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h"); + private static final HashSet RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h", "version"); @Inject @ConfigProperty(name = "dw.model.defaultTableName", defaultValue = DEFAULT_MODEL_TABLE_NAME) @@ -127,6 +132,32 @@ public class ModelBean { @GZIP @Interceptors(ResponseInterceptor.class) public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName) { + return listModelNames(modelTableName, false); + } + + /** + * Administrator credentials required. Get the names of the models, optionally bypassing the local model cache in order to avoid reading + * stale data + * + * + * @param modelTableName + * name of the table that contains the model + * @param skipCache + * if true, bypasses reads of cached model data, forcing a read from the data source instead + * @return datawave.webservice.model.ModelList + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 500 internal server error + */ + @GET + @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff", "text/html"}) + @Path("admin/list") + @GZIP + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Interceptors(ResponseInterceptor.class) + public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName, @QueryParam("skipCache") @DefaultValue("true") boolean skipCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -152,7 +183,7 @@ public ModelList listModelNames(@QueryParam("modelTableName") String modelTableN try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); - try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) { + try (Scanner scanner = createScanner(modelTableName, cbAuths, client, skipCache)) { for (Entry entry : scanner) { String colf = entry.getKey().getColumnFamily().toString(); if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) { @@ -184,28 +215,35 @@ else if (parts.length == 2) } /** - * Administrator credentials required. Insert a new model + * Administrator credentials required. Insert a new model or replace an existing one * * @param model * the model + * @param modelName + * name of the model. If non-empty, update/replace (PUT) semantics are enabled. Otherwise, it is assumed that user intends the model to be newly + * created (POST), returning an error response if the model already exists * @param modelTableName * name of the table that contains the model + * @param skipCache + * if true, bypasses reads of cached model data, forcing a read from the data source instead * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * * @HTTP 200 success - * @HTTP 412 if model already exists with this name, delete it first + * @HTTP 412 if model already exists. Delete it first, or retry with 'import/{modelName}' URI to enable update/replace * @HTTP 500 internal server error */ + @PUT @POST @Consumes({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml"}) @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - @Path("/import") + @Path("/import{pathDelimiter: (/)?}{modelName: ((?<=/)[a-zA-Z0-9_]*)?}") @GZIP @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors(ResponseInterceptor.class) - public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName) { + public VoidResponse importModel(datawave.webservice.model.Model model, @PathParam("modelName") String modelName, + @QueryParam("modelTableName") String modelTableName, @QueryParam("skipCache") @DefaultValue("true") boolean skipCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -214,14 +252,26 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar if (log.isDebugEnabled()) { log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); } + VoidResponse response = new VoidResponse(); + ModelList models = listModelNames(modelTableName, skipCache); - ModelList models = listModelNames(modelTableName); - if (models.getNames().contains(model.getName())) - throw new PreConditionFailedException(null, response); + if (StringUtils.isEmpty(modelName)) { + // Caller used '/import', so we enforce create-only + if (models.getNames().contains(model.getName())) { + throw new PreConditionFailedException(null, response); + } + } else { + // Caller used '/import/{modelName}', so we allow update/replace (and create) + if (!modelName.equals(model.getName())) { + throw new PreConditionFailedException(null, response); + } + if (models.getNames().contains(modelName)) { + deleteModel(modelName, modelTableName, skipCache); + } + } insertMapping(model, modelTableName); - return response; } @@ -232,6 +282,8 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar * model name to delete * @param modelTableName * name of the table that contains the model + * @param skipCache + * if true, bypasses reads of cached model data, forcing a read from the data source instead * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * @@ -246,27 +298,28 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar @GZIP @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) - public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) { + public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName, + @QueryParam("skipCache") @DefaultValue("true") boolean skipCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; } - return deleteModel(name, modelTableName, true); + return deleteModel(name, modelTableName, true, skipCache); } - private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache) { + private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache, boolean skipCache) { if (log.isDebugEnabled()) { log.debug("model name: " + name); log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); } VoidResponse response = new VoidResponse(); - ModelList models = listModelNames(modelTableName); + ModelList models = listModelNames(modelTableName, skipCache); if (!models.getNames().contains(name)) throw new NotFoundException(null, response); - datawave.webservice.model.Model model = getModel(name, modelTableName); + datawave.webservice.model.Model model = getModel(name, modelTableName, skipCache); deleteMapping(model, modelTableName, reloadCache); return response; @@ -281,6 +334,8 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl * name of copied model * @param modelTableName * name of the table that contains the model + * @param skipCache + * if true, bypasses reads of cached model data, forcing a read from the data source instead * @return datawave.webservice.result.VoidResponse * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user * @@ -296,17 +351,17 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @Required("newName") @FormParam("newName") String newName, - @FormParam("modelTableName") String modelTableName) { + @FormParam("modelTableName") String modelTableName, @FormParam("skipCache") @DefaultValue("true") boolean skipCache) { VoidResponse response = new VoidResponse(); if (modelTableName == null) { modelTableName = defaultModelTableName; } - datawave.webservice.model.Model model = getModel(name, modelTableName); + datawave.webservice.model.Model model = getModel(name, modelTableName, skipCache); // Set the new name model.setName(newName); - importModel(model, modelTableName); + importModel(model, newName, modelTableName, skipCache); return response; } @@ -331,6 +386,35 @@ public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @GZIP @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) { + return getModel(name, modelTableName, false); + } + + /** + * Administrator credentials required. Retrieve the model and all of its mappings, optionally bypassing the local model cache in order to + * avoid reading stale data + * + * @param name + * model name + * @param modelTableName + * name of the table that contains the model + * @param skipCache + * if true, bypasses reads of cached model data, forcing a read from the data source instead + * @return datawave.webservice.model.Model + * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user + * + * @HTTP 200 success + * @HTTP 404 model not found + * @HTTP 500 internal server error + */ + @GET + @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff", "text/html"}) + @Path("admin/{name}") + @GZIP + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) + public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName, + @QueryParam("skipCache") @DefaultValue("true") boolean skipCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -355,7 +439,7 @@ public datawave.webservice.model.Model getModel(@Required("name") @PathParam("na try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); - try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) { + try (Scanner scanner = createScanner(modelTableName, cbAuths, client, skipCache)) { IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); scanner.addScanIterator(cfg); @@ -543,4 +627,27 @@ private String checkModelTableName(String tableName) { else return tableName; } + + /** + * Scanner factory method + * + * @param tableName + * the table name + * @param auths + * the scanner auths + * @param client + * the AccumuloClient instance + * @param skipCache + * if true, forces a read of tableName, bypassing the webservice's internal cache. Ignored when the client is anything other than + * {@link WrappedAccumuloClient} + * @return + * @throws TableNotFoundException + */ + private Scanner createScanner(String tableName, Set auths, AccumuloClient client, boolean skipCache) throws TableNotFoundException { + if (client instanceof WrappedAccumuloClient) { + return WrappedScannerHelper.createScanner((WrappedAccumuloClient) client, this.checkModelTableName(tableName), auths, skipCache); + } else { + return ScannerHelper.createScanner(client, this.checkModelTableName(tableName), auths); + } + } } diff --git a/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java b/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java index 721c4ba0f62..879ba6e212c 100644 --- a/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java +++ b/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java @@ -139,7 +139,7 @@ public void testModelImportNoTable() throws Exception { EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); PowerMock.replayAll(); - bean.importModel(MODEL_ONE, (String) null); + bean.importModel(MODEL_ONE, (String) null, (String) null, false); PowerMock.verifyAll(); } @@ -160,7 +160,7 @@ private void importModels() throws Exception { EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); PowerMock.replayAll(); - bean.importModel(MODEL_ONE, (String) null); + bean.importModel(MODEL_ONE, (String) null, (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); @@ -178,7 +178,7 @@ private void importModels() throws Exception { EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); PowerMock.replayAll(); - bean.importModel(MODEL_TWO, (String) null); + bean.importModel(MODEL_TWO, (String) null, (String) null, false); PowerMock.verifyAll(); } @@ -247,7 +247,7 @@ public void testModelDelete() throws Exception { EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); PowerMock.replayAll(); - bean.deleteModel(MODEL_TWO.getName(), (String) null); + bean.deleteModel(MODEL_TWO.getName(), (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); @@ -318,7 +318,7 @@ public void testCloneModel() throws Exception { EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); PowerMock.replayAll(); - bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null); + bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);