Improve performance of AvscSchemaWriter (#537)

Co-authored-by: Karthik Ramgopal <[email protected]>
linkedin · Jan 18, 2024 · 42ca14f · 42ca14f
1 parent 62510e8
commit 42ca14f
Show file tree

Hide file tree

Showing 3 changed files with 210 additions and 153 deletions.
diff --git a/...uilder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java b/...uilder-spi/src/main/java/com/linkedin/avroutil1/builder/plugins/BuilderPluginContext.java
@@ -45,19 +45,27 @@ public void run() throws Exception {
       throw new IllegalStateException("run() has already been invoked");
     }
 
-    //"seal" any internal state to prevent plugins from trying to do weird things during execution
+    // "seal" any internal state to prevent plugins from trying to do weird things during execution
     sealed = true;
 
-    int operationCount = operations.stream().collect(StreamUtil.toParallelStream(op -> {
-      try {
-        op.run(operationContext);
-      } catch (Exception e) {
-        throw new IllegalStateException("Exception running operation", e);
-      }
+    if (!operations.isEmpty()) {
+      long operationStart = System.currentTimeMillis();
+      final int parallelism = Math.min(operations.size(), 5);
+      int operationCount = operations.stream().collect(StreamUtil.toParallelStream(op -> {
+        try {
+          op.run(operationContext);
+        } catch (Exception e) {
+          throw new IllegalStateException("Exception running operation", e);
+        }
 
-      return 1;
-    }, 2)).reduce(0, Integer::sum);
+        return 1;
+      }, parallelism)).reduce(0, Integer::sum);
 
-    LOGGER.info("Executed {} operations for builder plugins", operationCount);
+      long operationEnd = System.currentTimeMillis();
+      LOGGER.info("Executed {} operations with parallelism of {} for builder plugins in {} millis", operationCount,
+          parallelism, operationEnd - operationStart);
+    } else {
+      LOGGER.info("No operations specified to run");
+    }
   }
 }
diff --git a/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java b/avro-builder/builder/src/main/java/com/linkedin/avroutil1/builder/SchemaBuilder.java
@@ -28,20 +28,29 @@
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
  * generates java code from avsc files
  */
 public class SchemaBuilder {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(SchemaBuilder.class);
+
   private SchemaBuilder() { }
 
   public static void main(String[] args) throws Exception {
+    long start = System.currentTimeMillis();
+
+    long pluginLoadStart = System.currentTimeMillis();
     List<BuilderPlugin> plugins = loadPlugins(1);
+    long pluginLoadEnd = System.currentTimeMillis();
+    LOGGER.info("Loaded {} plugins in {} millis.", plugins.size(), pluginLoadEnd - pluginLoadStart);
 
+    long optionParseStart = System.currentTimeMillis();
     OptionParser parser = new OptionParser();
-
     OptionSpec<String> inputOpt = parser.accepts("input", "Schema or directory of schemas to compile [REQUIRED]")
         .withRequiredArg().required()
         .describedAs("file");
@@ -253,7 +262,10 @@ public static void main(String[] args) throws Exception {
     );
 
     opConfig.validateParameters();
+    long optionParseEnd = System.currentTimeMillis();
+    LOGGER.info("Parsed all options in {} millis.", optionParseEnd - optionParseStart);
 
+    long operationContextBuildStart = System.currentTimeMillis();
     OperationContextBuilder operationContextBuilder;
     switch (opConfig.getGeneratorType()) {
       case AVRO_UTIL:
@@ -267,6 +279,9 @@ public static void main(String[] args) throws Exception {
         throw new IllegalStateException("unhandled: " + opConfig.getGeneratorType());
     }
     OperationContext opContext = operationContextBuilder.buildOperationContext(opConfig);
+    long operationContextBuildEnd = System.currentTimeMillis();
+    LOGGER.info("Built operation context in {} millis.", operationContextBuildStart - operationContextBuildEnd);
+
     BuilderPluginContext context = new BuilderPluginContext(opContext);
 
     // Allow other plugins to add operations
@@ -275,6 +290,9 @@ public static void main(String[] args) throws Exception {
     }
 
     context.run();
+
+    long end = System.currentTimeMillis();
+    LOGGER.info("Finished running SchemaBuilder in {} millis", end - start);
   }
 
   private static List<BuilderPlugin> loadPlugins(@SuppressWarnings("SameParameterValue") int currentApiVersion) {