Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option in SchemaBuilder to ignore user defined json properties during codegen. #542

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.stream.Collectors;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
Expand Down Expand Up @@ -114,6 +116,12 @@ public static void main(String[] args) throws Exception {
.defaultsTo("false")
.describedAs("true/false");

OptionSpec<String> jsonPropsToIgnoreInCompareOpt = parser.accepts("jsonPropsToIgnoreInCompare",
"json properties to ignore when comparing schemas on filesystem and classpath. Example : jsonOption1, jsonOption2")
.withOptionalArg()
.defaultsTo("")
.withValuesSeparatedBy(',');

//allow plugins to add CLI options
for (BuilderPlugin plugin : plugins) {
plugin.customizeCLI(parser);
Expand Down Expand Up @@ -238,6 +246,12 @@ public static void main(String[] args) throws Exception {
skipCodegenIfSchemaOnClasspath = Boolean.TRUE.equals(Boolean.parseBoolean(value));
}

Set<String> jsonPropsToIgnoreInCompare = Collections.EMPTY_SET;
if (options.has(jsonPropsToIgnoreInCompareOpt)) {
jsonPropsToIgnoreInCompare =
options.valuesOf(jsonPropsToIgnoreInCompareOpt).stream().map(String::trim).collect(Collectors.toSet());
}

//allow plugins to parse and validate their own added options
for (BuilderPlugin plugin : plugins) {
plugin.parseAndValidateOptions(options);
Expand All @@ -258,7 +272,8 @@ public static void main(String[] args) throws Exception {
minAvroVer,
handleAvro702,
handleUtf8EncodingInPutByIndex,
skipCodegenIfSchemaOnClasspath
skipCodegenIfSchemaOnClasspath,
jsonPropsToIgnoreInCompare
);

opConfig.validateParameters();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;


/**
Expand Down Expand Up @@ -49,6 +50,10 @@ public class CodeGenOpConfig {
boolean avro702Handling;
boolean utf8EncodingPutByIndex;
boolean skipCodegenIfSchemaOnClasspath;
/**
* List of JSON properties to ignore when comparing two schemas during codegen.
*/
private Set<String> jsonPropsToIgnoreInCompare;

@Deprecated
public CodeGenOpConfig(
Expand Down Expand Up @@ -201,6 +206,40 @@ public CodeGenOpConfig(List<File> inputRoots,
this.skipCodegenIfSchemaOnClasspath = skipCodegenIfSchemaOnClasspath;
}

public CodeGenOpConfig(List<File> inputRoots,
List<File> nonImportableSourceRoots,
boolean includeClasspath,
File outputSpecificRecordClassesRoot,
File outputExpandedSchemasRoot,
List<File> resolverPath,
CodeGenerator generatorType,
DuplicateSchemaBehaviour dupBehaviour,
List<String> duplicateSchemasToIgnore,
StringRepresentation stringRepresentation,
StringRepresentation methodStringRepresentation,
AvroVersion minAvroVersion,
boolean avro702Handling,
boolean handleUtf8EncodingInPutByIndex,
boolean skipCodegenIfSchemaOnClasspath,
Set<String> jsonPropsToIgnoreInCompare) {
this.inputRoots = inputRoots;
this.nonImportableSourceRoots = nonImportableSourceRoots;
this.includeClasspath = includeClasspath;
this.outputSpecificRecordClassesRoot = outputSpecificRecordClassesRoot;
this.outputExpandedSchemasRoot = outputExpandedSchemasRoot;
this.resolverPath = resolverPath;
this.generatorType = generatorType;
this.dupBehaviour = dupBehaviour;
this.duplicateSchemasToIgnore = duplicateSchemasToIgnore;
this.stringRepresentation = stringRepresentation;
this.methodStringRepresentation = methodStringRepresentation;
this.minAvroVersion = minAvroVersion;
this.avro702Handling = avro702Handling;
this.utf8EncodingPutByIndex = handleUtf8EncodingInPutByIndex;
this.skipCodegenIfSchemaOnClasspath = skipCodegenIfSchemaOnClasspath;
this.jsonPropsToIgnoreInCompare = jsonPropsToIgnoreInCompare;
}

/**
* validates all input parameters (set at construction time)
*/
Expand Down Expand Up @@ -312,6 +351,10 @@ public boolean shouldSkipCodegenIfSchemaOnClasspath() {
return skipCodegenIfSchemaOnClasspath;
}

public Set<String> getJsonPropsToIgnoreInCompare() {
return jsonPropsToIgnoreInCompare;
}

private void validateInput(Collection<File> files, String desc) {
for (File f : files) {
if (!f.exists()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,10 @@ public OperationContext buildOperationContext(CodeGenOpConfig config) throws Exc
// check if the schema on classpath is the same as the one we are trying to generate
AvroSchema avroSchemaFromClasspath = (new AvscParser()).parse(cpSchema.toString()).getTopLevelSchema();
boolean areEqual = ConfigurableAvroSchemaComparator.equals(avroSchemaFromClasspath, schema,
SchemaComparisonConfiguration.STRICT);
SchemaComparisonConfiguration.newBuilder(SchemaComparisonConfiguration.STRICT).setJsonPropNamesToIgnore(
config.getJsonPropsToIgnoreInCompare()).build());
if (!areEqual) {

throw new IllegalStateException("Schema with name " + fullName
+ " is defined in the filesystem and on the classpath, but the two schemas are not equal.");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,29 @@ public void testTopLevelUnionUsingOwnCodegen() throws Exception {
Assert.assertEquals(javaFiles.size(), 2);
}

@Test
public void testTopLevelUnionUsingOwnCodegenWithIngoreJsonPropOptipn() throws Exception {
File simpleProjectRoot = new File(locateTestProjectsRoot(), "union-schema-project");
File inputFolder = new File(simpleProjectRoot, "input");
File outputFolder = new File(simpleProjectRoot, "output");
if (outputFolder.exists()) { //clear output
FileUtils.deleteDirectory(outputFolder);
}
//run the builder
SchemaBuilder.main(new String[] {
"--input", inputFolder.getAbsolutePath(),
"--output", outputFolder.getAbsolutePath(),
"--generator", CodeGenerator.AVRO_UTIL.name(),
"enableUtf8EncodingInPutByIndex", "false",
"--jsonPropsToIgnoreInCompare", "option1, option2"
});
//see output was generated
List<Path> javaFiles = Files.find(outputFolder.toPath(), 5,
(path, basicFileAttributes) -> path.getFileName().toString().endsWith(".java")
).collect(Collectors.toList());
Assert.assertEquals(javaFiles.size(), 2);
}

private File locateTestProjectsRoot() {
//the current working directory for test execution varies across gradle and IDEs.
//as such, we need to get creative to locate the folder
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ public Set<String> getJsonPropNamesToIgnore() {
return jsonPropNamesToIgnore;
}

// Builder class


public SchemaComparisonConfiguration compareStringJsonProps(boolean compare) {
return new SchemaComparisonConfiguration(
compare,
Expand Down Expand Up @@ -164,6 +167,86 @@ public SchemaComparisonConfiguration jsonPropNamesToIgnore(Set<String> jsonPropN
jsonPropNamesToIgnore
);
}

/**
* Get instance of SchemaComparisonConfiguration.Builder
* @param schemaConfig SchemaComparisonConfiguration
* @return Builder instance
*/
public static Builder newBuilder(SchemaComparisonConfiguration schemaConfig) {
return new Builder().from(schemaConfig);
}

/**
* Builder class for {@link SchemaComparisonConfiguration}
*/
public static class Builder {
private boolean compareStringJsonProps;
private boolean compareNonStringJsonProps;
private boolean compareAliases;
private boolean compareIntToFloatDefaults;
private boolean compareFieldOrder;
private boolean compareFieldLogicalTypes;
private Set<String> jsonPropNamesToIgnore;

public Builder setCompareStringJsonProps(boolean compareStringJsonProps) {
this.compareStringJsonProps = compareStringJsonProps;
return this;
}

public Builder setCompareNonStringJsonProps(boolean compareNonStringJsonProps) {
this.compareNonStringJsonProps = compareNonStringJsonProps;
return this;
}

public Builder setCompareAliases(boolean compareAliases) {
this.compareAliases = compareAliases;
return this;
}

public Builder setCompareIntToFloatDefaults(boolean compareIntToFloatDefaults) {
this.compareIntToFloatDefaults = compareIntToFloatDefaults;
return this;
}

public Builder setCompareFieldOrder(boolean compareFieldOrder) {
this.compareFieldOrder = compareFieldOrder;
return this;
}

public Builder setCompareFieldLogicalTypes(boolean compareFieldLogicalTypes) {
this.compareFieldLogicalTypes = compareFieldLogicalTypes;
return this;
}

public Builder setJsonPropNamesToIgnore(Set<String> jsonPropNamesToIgnore) {
this.jsonPropNamesToIgnore = jsonPropNamesToIgnore;
return this;
}

Builder from(SchemaComparisonConfiguration config) {
this.compareStringJsonProps = config.isCompareStringJsonProps();
this.compareNonStringJsonProps = config.isCompareNonStringJsonProps();
this.compareAliases = config.isCompareAliases();
this.compareIntToFloatDefaults = config.isCompareIntToFloatDefaults();
this.compareFieldOrder = config.isCompareFieldOrder();
this.compareFieldLogicalTypes = config.isCompareFieldLogicalTypes();
this.jsonPropNamesToIgnore = config.getJsonPropNamesToIgnore();
return this;
}

public SchemaComparisonConfiguration build() {
return new SchemaComparisonConfiguration(
compareStringJsonProps,
compareNonStringJsonProps,
compareAliases,
compareIntToFloatDefaults,
compareFieldOrder,
compareFieldLogicalTypes,
jsonPropNamesToIgnore
);
}
}
}


Loading