Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Barrage: Version Bumps, Feat Deprecation, New Feat Flags #41

Merged
merged 11 commits into from
Nov 11, 2024
Merged

Barrage: Version Bumps, Feat Deprecation, New Feat Flags #41

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6cc2dfd
Barrage: Version Bumps, Feat Deprecation, New Feat Flags
nbauernfeind Nov 4, 2024
699da10
Generate java flatbuffer results and add to repository
nbauernfeind Nov 4, 2024
4b99a4e
Fixup Build
nbauernfeind Nov 5, 2024
4b92d59
Fixup mvn install
nbauernfeind Nov 5, 2024
3b25f2f
Update java version
nbauernfeind Nov 5, 2024
b3490d7
Update java/format/README.md
nbauernfeind Nov 5, 2024
dc8195b
Proposed changes to the update (#42)
niloc132 Nov 6, 2024
2f884a5
Make header a year range
nbauernfeind Nov 6, 2024
e7a79c4
Ryan's feedback from dh-core text
nbauernfeind Nov 9, 2024
0ccdb38
Setup npm for 0.7.0 release
nbauernfeind Nov 9, 2024
bc36b11
Colin's feedback
nbauernfeind Nov 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
- name: Setup JDK
uses: actions/setup-java@v1
with:
java-version: '8.0.282'
java-version: '17.0.11'

- name: Package with Maven
run: mvn --file java/pom.xml --batch-mode package
run: mvn --file java/pom.xml --batch-mode package
62 changes: 23 additions & 39 deletions format/Barrage.fbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,7 @@ enum BarrageMessageType : byte {
/// if the msg_payload is empty.
None = 0,

/// for session management (not-yet-used)
NewSessionRequest = 1,
RefreshSessionRequest = 2,
SessionInfoResponse = 3,
/// enum values 1 - 3 are reserved for future use
nbauernfeind marked this conversation as resolved.
Show resolved Hide resolved

/// for subscription parsing/management (aka DoPut, DoExchange)
BarrageSerializationOptions = 4,
Expand All @@ -47,42 +44,13 @@ table BarrageMessageWrapper {
msg_payload: [byte];
}

/// Establish a new session.
table NewSessionRequest {
/// A nested protocol version (gets delegated to handshake)
protocol_version: uint;

/// Arbitrary auth/handshake info.
payload: [byte];
}

/// Refresh the provided session.
table RefreshSessionRequest {
/// this session token is only required if it is the first request of a handshake rpc stream
session: [byte];
}

/// Information about the current session state.
table SessionInfoResponse {
/// this is the metadata header to identify this session with future requests; it must be lower-case and remain static for the life of the session
metadata_header: [byte];

/// this is the session_token; note that it may rotate
session_token: [byte];

/// a suggested time for the user to refresh the session if they do not do so earlier; value is denoted in milliseconds since epoch
token_refresh_deadline_ms: long;
}

/// There will always be types that cannot be easily supported over IPC. These are the options:
/// Stringify (default) - Pretend the column is a string when sending over Arrow Flight (default)
/// JavaSerialization - Use java serialization; the client is responsible for the deserialization
/// ThrowError - Refuse to send the column and throw an internal error sharing as much detail as possible
/// There will always be types that cannot be easily supported over IPC. While column conversion mode is no longer
/// supported, users can more explicitly configure the encoding/decoding behavior of the server.
enum ColumnConversionMode : byte { Stringify = 1, JavaSerialization, ThrowError }

table BarrageSubscriptionOptions {
/// see enum for details
column_conversion_mode: ColumnConversionMode = Stringify;
/// Column conversion mode is no longer supported.
column_conversion_mode: ColumnConversionMode = Stringify (deprecated);

/// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
/// by eliminating the additional complexity of the validity buffer.
Expand All @@ -109,6 +77,12 @@ table BarrageSubscriptionOptions {
/// If true, the server will wrap columns with a list. This is useful for clients that do not support modified batches
/// with columns of differing lengths.
columns_as_list: bool;

/// If zero, list lengths will not be limited. If greater than zero, the server will limit the length of any encoded
/// list / array to the first n elements, where n is the specified value. If the column value has length less than
/// zero, the server will send the last n elements of the list / array. If the column value has length greater than
/// the limit, the server will encode the list up to the limit and append an arbitrary value to indicate truncation.
preview_list_length_limit: int = 0;
nbauernfeind marked this conversation as resolved.
Show resolved Hide resolved
}

/// Describes the subscription the client would like to acquire.
Expand All @@ -129,11 +103,15 @@ table BarrageSubscriptionRequest {
/// When this is set the viewport RowSet will be inverted against the length of the table. That is to say
/// every position value is converted from `i` to `n - i - 1` if the table has `n` rows.
reverse_viewport: bool;

/// If this is set, the server will parrot this subscription token in the response. This token can be used to identify
/// which subscription the server is now respecting.
subscription_token: [byte];
}

table BarrageSnapshotOptions {
/// see enum for details
column_conversion_mode: ColumnConversionMode = Stringify;
/// Column conversion mode is no longer supported.
column_conversion_mode: ColumnConversionMode = Stringify (deprecated);

/// Deephaven reserves a value in the range of primitives as a custom NULL value. This enables more efficient transmission
/// by eliminating the additional complexity of the validity buffer.
Expand All @@ -148,6 +126,12 @@ table BarrageSnapshotOptions {
/// of one row per batch). If the message size limit cannot be met due to large row sizes, the server will throw a
/// `Status.RESOURCE_EXHAUSTED` exception
max_message_size: int;

/// If zero, list lengths will not be limited. If greater than zero, the server will limit the length of any encoded
/// list / array to the first n elements, where n is the specified value. If the column value has length less than
/// zero, the server will send the last n elements of the list / array. If the column value has length greater than
/// the limit, the server will encode the list up to the limit and append an arbitrary value to indicate truncation.
preview_list_length_limit: int = 0;
}

/// Describes the snapshot the client would like to acquire.
Expand Down
26 changes: 21 additions & 5 deletions java/barrage-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<parent>
<groupId>io.deephaven.barrage</groupId>
<artifactId>barrage-java-root</artifactId>
<version>0.6.1-SNAPSHOT</version>
<version>0.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand All @@ -29,8 +29,9 @@
<packaging>jar</packaging>

<properties>
<dep.grpc.version>1.58.0</dep.grpc.version>
<dep.protobuf.version>3.21.9</dep.protobuf.version>
<dep.grpc.version>1.65.0</dep.grpc.version>
<dep.protobuf.version>3.25.4</dep.protobuf.version>
<dep.guava-bom.version>33.3.1-jre</dep.guava-bom.version>
<forkCount>1</forkCount>
</properties>

Expand All @@ -45,6 +46,10 @@
<artifactId>grpc-protobuf</artifactId>
<version>${dep.grpc.version}</version>
</dependency>
<dependency>
niloc132 marked this conversation as resolved.
Show resolved Hide resolved
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-stub</artifactId>
Expand Down Expand Up @@ -72,6 +77,17 @@
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava-bom</artifactId>
<version>${dep.guava-bom.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
niloc132 marked this conversation as resolved.
Show resolved Hide resolved
<build>
<extensions>
<extension>
Expand Down Expand Up @@ -184,8 +200,8 @@
</goals>
<configuration>
<ignoredDependencies>
<ignoredDependency>io.grpc:grpc-core:jar:1.30.2</ignoredDependency>
<ignoredDependency>io.grpc:grpc-context:jar:1.30.2</ignoredDependency>
<ignoredDependency>io.grpc:grpc-core:jar:${dep.grpc.version}</ignoredDependency>
<ignoredDependency>io.grpc:grpc-context:jar:${dep.grpc.version}</ignoredDependency>
</ignoredDependencies>
</configuration>
</execution>
Expand Down
32 changes: 32 additions & 0 deletions java/format/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
### Updating the flatbuffers generated code

1. Verify that your version of flatc matches the declared dependency:

```bash
$ flatc --version
flatc version 24.3.25
$ grep "dep.fbs.version" java/pom.xml
<dep.fbs.version>1.12.0</dep.fbs.version>
nbauernfeind marked this conversation as resolved.
Show resolved Hide resolved
```

2. Generate the flatbuffer java files by performing the following:

```bash
cd $BARRAGE_HOME
# remove the existing files
rm -rf java/format/src
# regenerate from the .fbs files
flatc --java -o java/format/src/main/java format/*.fbs
# prepend license header
find java/format/src -type f | while read file; do
(cat header | while read line; do echo "// $line"; done; cat $file) > $file.tmp
mv $file.tmp $file
niloc132 marked this conversation as resolved.
Show resolved Hide resolved
done
```

3. Ensure any new files are added to the git repository:

```bash
cd $BARRAGE_HOME
find java/format/src -type f | while read file; do git add $file; done
```
124 changes: 1 addition & 123 deletions java/format/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,141 +20,19 @@
<parent>
<artifactId>barrage-java-root</artifactId>
<groupId>io.deephaven.barrage</groupId>
<version>0.6.1-SNAPSHOT</version>
<version>0.7.0-SNAPSHOT</version>
</parent>

<artifactId>barrage-format</artifactId>
<packaging>jar</packaging>
<name>Barrage Format</name>
<description>Generated Java files from the IPC Flatbuffer definitions.</description>

<properties>
<flatc.download.skip>false</flatc.download.skip>
<flatc.executable>${project.build.directory}/flatc-${os.detected.classifier}-${dep.flatc.version}.exe</flatc.executable>
<flatc.generated.files>${project.build.directory}/generated-sources/flatc</flatc.generated.files>
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
</properties>

<dependencies>
<dependency>
<groupId>com.google.flatbuffers</groupId>
<artifactId>flatbuffers-java</artifactId>
</dependency>
</dependencies>

<build>
<extensions>
<!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
<extension>
<groupId>kr.motd.maven</groupId>
<artifactId>os-maven-plugin</artifactId>
<version>${os-maven-plugin.version}</version>
</extension>
</extensions>

<plugins>
<plugin> <!-- download the flatbuffer compiler -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-flatc</id>
<phase>initialize</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>com.github.shinanca</groupId>
<artifactId>flatc-${os.detected.classifier}</artifactId>
<version>${dep.flatc.version}</version>
<type>exe</type>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}</outputDirectory>
</artifactItem>
</artifactItems>
<skip>${flatc.download.skip}</skip>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.4.0</version>
<executions>
<execution> <!-- make the flatbuffer compiler executable -->
<id>script-chmod</id>
<goals>
<goal>exec</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<executable>chmod</executable>
<arguments>
<argument>+x</argument>
<argument>${project.build.directory}/flatc-${os.detected.classifier}-${dep.flatc.version}.exe</argument>
</arguments>
<skip>${flatc.download.skip}</skip>
</configuration>
</execution>
<execution> <!-- generate sources by executing the flatbuffer compiler -->
<goals>
<goal>exec</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<executable>${flatc.executable}</executable>
<arguments>
<argument>-j</argument>
<argument>-o</argument>
<argument>${flatc.generated.files}</argument>
<argument>../../format/Barrage.fbs</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin> <!-- add the license header to the generated files -->
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>2.3</version>
<configuration>
<header>${basedir}/../../header</header>
<includes>
<include>**/*.java</include>
</includes>
</configuration>
<executions>
<execution>
<phase>process-sources</phase>
<goals>
<goal>format</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin> <!-- add generated sources to classpath -->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.9.1</version>
<executions>
<execution>
<id>add-generated-sources-to-classpath</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${flatc.generated.files}</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>

</build>
</project>
40 changes: 40 additions & 0 deletions java/format/src/main/java/io/deephaven/barrage/flatbuf/BarrageMessageType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2020 Deephaven Data Labs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// automatically generated by the FlatBuffers compiler, do not modify

package io.deephaven.barrage.flatbuf;

@SuppressWarnings("unused")
public final class BarrageMessageType {
private BarrageMessageType() { }
/**
* A barrage message wrapper might send a None message type
* if the msg_payload is empty.
*/
public static final byte None = 0;
/**
* enum values 1 - 3 are reserved for future use
* for subscription parsing/management (aka DoPut, DoExchange)
*/
public static final byte BarrageSerializationOptions = 4;
public static final byte BarrageSubscriptionRequest = 5;
public static final byte BarrageUpdateMetadata = 6;
public static final byte BarrageSnapshotRequest = 7;
public static final byte BarragePublicationRequest = 8;

public static final String[] names = { "None", "", "", "", "BarrageSerializationOptions", "BarrageSubscriptionRequest", "BarrageUpdateMetadata", "BarrageSnapshotRequest", "BarragePublicationRequest", };

public static String name(int e) { return names[e]; }
}

Loading
Loading