Barrage: Version Bumps, Feat Deprecation, New Feat Flags

.github/workflows/package.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Setup JDK
         uses: actions/setup-java@v1
         with:
-          java-version: '8.0.282'
+          java-version: '17.0.11'
 
       - name: Package with Maven
-        run: mvn --file java/pom.xml --batch-mode package
+        run: mvn --file java/pom.xml --batch-mode package

format/Barrage.fbs

@@ -19,10 +19,10 @@ enum BarrageMessageType : byte {
   /// if the msg_payload is empty.
   None = 0,
 
-  /// for session management (not-yet-used)
-  NewSessionRequest = 1,
-  RefreshSessionRequest = 2,
-  SessionInfoResponse = 3,
+  /// enum values 1 - 3 are reserved for future use
+  UNUSED_1 = 1,
+  UNUSED_2 = 2,
+  UNUSED_3 = 3,
 
   /// for subscription parsing/management (aka DoPut, DoExchange)
   BarrageSerializationOptions = 4,
@@ -47,42 +47,13 @@ table BarrageMessageWrapper {
   msg_payload: [byte];
 }
 
-/// Establish a new session.
-table NewSessionRequest {
-  /// A nested protocol version (gets delegated to handshake)
-  protocol_version: uint;
-
-  /// Arbitrary auth/handshake info.
-  payload: [byte];
-}
-
-/// Refresh the provided session.
-table RefreshSessionRequest {
-  /// this session token is only required if it is the first request of a handshake rpc stream
-  session: [byte];
-}
-
-/// Information about the current session state.
-table SessionInfoResponse {
-  /// this is the metadata header to identify this session with future requests; it must be lower-case and remain static for the life of the session
-  metadata_header: [byte];
-
-  /// this is the session_token; note that it may rotate
-  session_token: [byte];
-
-  /// a suggested time for the user to refresh the session if they do not do so earlier; value is denoted in milliseconds since epoch
-  token_refresh_deadline_ms: long;
-}
-
-/// There will always be types that cannot be easily supported over IPC. These are the options:
-///   Stringify (default) - Pretend the column is a string when sending over Arrow Flight (default)
-///   JavaSerialization   - Use java serialization; the client is responsible for the deserialization
-///   ThrowError          - Refuse to send the column and throw an internal error sharing as much detail as possible
+/// There will always be types that cannot be easily supported over IPC. While column conversion mode is no longer
+/// supported, users can more explicitly configure the encoding/decoding behavior of the server.
 enum ColumnConversionMode : byte { Stringify = 1, JavaSerialization, ThrowError }
 
 table BarrageSubscriptionOptions {
-  /// see enum for details
-  column_conversion_mode: ColumnConversionMode = Stringify;
+  /// Column conversion mode is no longer supported.
+  column_conversion_mode: ColumnConversionMode = Stringify (deprecated);
 
   /// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
   /// by eliminating the additional complexity of the validity buffer.
@@ -109,6 +80,15 @@ table BarrageSubscriptionOptions {
   /// If true, the server will wrap columns with a list. This is useful for clients that do not support modified batches
   /// with columns of differing lengths.
   columns_as_list: bool;
+
+  /// The maximum length of any list / array to encode.
+  /// - If zero, list lengths will not be limited.
+  /// - If non-zero, the server will limit the length of any encoded list / array to the absolute value of the returned length.
+  /// - If less than zero, the server will encode elements from the end of the list / array, rather than from the beginning.
+  ///
+  /// Note: The server is unable to indicate when truncation occurs. To detect truncation request one more element than
+  /// the maximum number you wish to display.
+  preview_list_length_limit: long = 0;
 }
 
 /// Describes the subscription the client would like to acquire.
@@ -129,11 +109,15 @@ table BarrageSubscriptionRequest {
   /// When this is set the viewport RowSet will be inverted against the length of the table. That is to say
   /// every position value is converted from `i` to `n - i - 1` if the table has `n` rows.
   reverse_viewport: bool;
+
+  /// If this is set, the server will parrot this subscription token in the response. This token can be used to identify
+  /// which subscription the server is now respecting.
+  subscription_token: [byte];
 }
 
 table BarrageSnapshotOptions {
-  /// see enum for details
-  column_conversion_mode: ColumnConversionMode = Stringify;
+  /// Column conversion mode is no longer supported.
+  column_conversion_mode: ColumnConversionMode = Stringify (deprecated);
 
   /// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
   /// by eliminating the additional complexity of the validity buffer.
@@ -148,6 +132,15 @@ table BarrageSnapshotOptions {
   /// of one row per batch). If the message size limit cannot be met due to large row sizes, the server will throw a
   /// `Status.RESOURCE_EXHAUSTED` exception
   max_message_size: int;
+
+  /// The maximum length of any list / array to encode.
+  /// - If zero, list lengths will not be limited.
+  /// - If non-zero, the server will limit the length of any encoded list / array to the absolute value of the returned length.
+  /// - If less than zero, the server will encode elements from the end of the list / array, rather than from the beginning.
+  ///
+  /// Note: The server is unable to indicate when truncation occurs. To detect truncation request one more element than
+  /// the maximum number you wish to display.
+  preview_list_length_limit: long = 0;
 }
 
 /// Describes the snapshot the client would like to acquire.

header

@@ -1,4 +1,4 @@
-Copyright 2020 Deephaven Data Labs
+Copyright 2020-2024 Deephaven Data Labs
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

java/barrage-core/pom.xml

@@ -19,31 +19,31 @@
   <parent>
     <groupId>io.deephaven.barrage</groupId>
     <artifactId>barrage-java-root</artifactId>
-    <version>0.6.1-SNAPSHOT</version>
+    <version>0.7.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>barrage-core</artifactId>
   <name>Deephaven Barrage Core</name>
   <description>An RPC mechanism for transferring ticking Arrow data.</description>
-  <packaging>jar</packaging>
 
   <properties>
-    <dep.grpc.version>1.58.0</dep.grpc.version>
-    <dep.protobuf.version>3.21.9</dep.protobuf.version>
-    <forkCount>1</forkCount>
+    <dep.grpc.version>1.65.0</dep.grpc.version>
+    <dep.protobuf.version>3.25.4</dep.protobuf.version>
+    <dep.guava.version>33.3.1-jre</dep.guava.version>
+    <protoOutDir>${project.build.directory}/generated-sources/protobuf</protoOutDir>
   </properties>
 
   <dependencies>
     <dependency>
       <groupId>io.grpc</groupId>
-      <artifactId>grpc-core</artifactId>
+      <artifactId>grpc-protobuf</artifactId>
       <version>${dep.grpc.version}</version>
     </dependency>
     <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-protobuf</artifactId>
-      <version>${dep.grpc.version}</version>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>${dep.guava.version}</version>
     </dependency>
     <dependency>
       <groupId>io.grpc</groupId>
@@ -61,15 +61,8 @@
       <version>${dep.grpc.version}</version>
     </dependency>
     <dependency>
-        <groupId>org.apache.arrow</groupId>
-        <artifactId>flight-core</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.api.grpc</groupId>
-      <artifactId>proto-google-common-protos</artifactId>
-      <version>1.12.0</version>
-      <scope>test</scope>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>flight-core</artifactId>
     </dependency>
   </dependencies>
   <build>
@@ -81,69 +74,6 @@
       </extension>
     </extensions>
     <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>3.4.0</version>
-        <executions>
-          <execution>
-            <id>shade-main</id>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadedArtifactAttached>true</shadedArtifactAttached>
-              <shadedClassifierName>shaded</shadedClassifierName>
-              <artifactSet>
-                <includes>
-                  <include>io.grpc:*</include>
-                  <include>com.google.protobuf:*</include>
-                </includes>
-              </artifactSet>
-              <relocations>
-                <relocation>
-                  <pattern>com.google.protobuf</pattern>
-                  <shadedPattern>io.deephaven.barrage.com.google.protobuf</shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-          <execution>
-            <id>shade-ext</id>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-            <configuration>
-              <shadedArtifactAttached>true</shadedArtifactAttached>
-              <shadedClassifierName>shaded-ext</shadedClassifierName>
-              <artifactSet>
-                <includes>
-                  <include>io.grpc:*</include>
-                  <include>com.google.protobuf:*</include>
-                </includes>
-              </artifactSet>
-              <relocations>
-                <relocation>
-                  <pattern>com.google.protobuf</pattern>
-                  <shadedPattern>io.deephaven.barrage.com.google.protobuf</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.google.common</pattern>
-                  <shadedPattern>io.deephaven.barrage.com.google.common</shadedPattern>
-                </relocation>
-              </relocations>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
-              </transformers>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <groupId>org.xolstice.maven.plugins</groupId>
         <artifactId>protobuf-maven-plugin</artifactId>
@@ -159,7 +89,7 @@
             <id>src</id>
             <configuration>
               <protoSourceRoot>${basedir}/../../format/</protoSourceRoot>
-              <outputDirectory>${project.build.directory}/generated-sources/protobuf</outputDirectory>
+              <outputDirectory>${protoOutDir}</outputDirectory>
               <includes>
                   <!-- Note we only want to include the BrowserFlight service! -->
                   <include>**/BrowserFlight.proto</include>
@@ -172,22 +102,24 @@
           </execution>
         </executions>
       </plugin>
+      <!-- add the license header to the generated files -->
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
+        <groupId>com.mycila</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+        <version>4.6</version>
+        <configuration>
+          <header>${basedir}/../../header</header>
+          <includes>
+            <include>**/*.java</include>
+          </includes>
+          <defaultBasedir>${protoOutDir}</defaultBasedir>
+        </configuration>
         <executions>
           <execution>
-            <id>analyze</id>
-            <phase>verify</phase>
+            <phase>process-sources</phase>
             <goals>
-              <goal>analyze-only</goal>
+              <goal>format</goal>
             </goals>
-            <configuration>
-                <ignoredDependencies>
-                    <ignoredDependency>io.grpc:grpc-core:jar:1.30.2</ignoredDependency>
-                    <ignoredDependency>io.grpc:grpc-context:jar:1.30.2</ignoredDependency>
-                </ignoredDependencies>
-            </configuration>
           </execution>
         </executions>
       </plugin>
@@ -204,30 +136,12 @@
             </goals>
             <configuration>
               <sources>
-                <source>${project.build.directory}/generated-sources/protobuf</source>
+                <source>${protoOutDir}</source>
               </sources>
             </configuration>
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <version>3.4.2</version>
-        <configuration>
-          <descriptorRefs>
-            <descriptorRef>jar-with-dependencies</descriptorRef>
-          </descriptorRefs>
-        </configuration>
-        <executions>
-          <execution>
-            <id>make-assembly</id>
-            <phase>package</phase>
-            <goals>
-              <goal>single</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 </project>

java/format/README.md

@@ -0,0 +1,29 @@
+### Updating the flatbuffers generated code
+
+1. Verify that your version of flatc matches the declared dependency:
+
+```bash
+$ flatc --version
+flatc version 24.3.25
+$ grep "dep.fbs.version" java/pom.xml
+    <dep.fbs.version>24.3.25</dep.fbs.version>
+```
+
+2. Generate the flatbuffer java files by performing the following:
+
+```bash
+cd $BARRAGE_HOME
+# remove the existing files
+rm -rf java/format/src
+# regenerate from the .fbs files
+flatc --java -o java/format/src/main/java format/*.fbs
+# generate license headers
+mvn compile
+```
+
+3. Ensure any new files are added to the git repository:
+
+```bash
+cd $BARRAGE_HOME
+find java/format/src -type f | while read file; do git add $file; done
+```