-
Notifications
You must be signed in to change notification settings - Fork 5
/
pom.xml
222 lines (220 loc) · 8.76 KB
/
pom.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.sungard.dataflow</groupId>
<artifactId>dataflow-whitepaper</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>dataflow-whitepaper</name>
<url>https://github.com/SunGard-Labs/dataflow-whitepaper</url>
<properties>
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>yyyy-MM-dd-HH-mm</maven.build.timestamp.format>
<gcp.project>sungard-cat-demo</gcp.project>
<gcp.stagingLocation>gs://dataflow-staging-test</gcp.stagingLocation>
<df.jobName>options-symbol-flow-${timestamp}</df.jobName>
<df.jobFile>/tmp/dataflow-job-file</df.jobFile>
<df.workerMachineType>n1-standard-4</df.workerMachineType>
<df.numWorkers>20</df.numWorkers>
<df.workerDiskSize>50</df.workerDiskSize>
<df.bqDest>dataflow_project.optionsdata</df.bqDest>
<df.gsc>gs://pick_a_bucket_name/</df.gsc>
<df.scalingAlgorithm>BASIC</df.scalingAlgorithm>
<df.input>${basedir}/input/zvzzt.input.txt</df.input>
<!-- <df.input>${df.gsc}/zvzzt.input.txt</df.input> -->
<df.outputFile>${basedir}/output/zvzzt.output.txt</df.outputFile>
<!-- <df.outputFile>${df.gsc}/zvzzt.output.txt</df.outputFile> -->
<df.outputTable>${gcp.project}:${df.bqDest}</df.outputTable>
<slf4j.version>1.7.7</slf4j.version>
<logback-classic.version>1.0.1</logback-classic.version>
<junit.version>4.10</junit.version>
<hamcrest.version>1.3</hamcrest.version>
<mockito-all.version>1.10.19</mockito-all.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.cloud.dataflow</groupId>
<artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
<version>LATEST</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback-classic.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito-all.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>${hamcrest.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.dataflow</groupId>
<artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.6</version>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<!-- include any jar that would normally be in the containers lib -->
<excludes>
<!-- <exclude>org.apache.storm:*</exclude> -->
<exclude>io.netty:*</exclude>
<exclude>org.objenesis:*</exclude>
<exclude>com.esotericsoftware.reflectasm:reflectasm</exclude>
<exclude>com.esotericsoftware:reflectasm</exclude>
<exclude>com.esotericsoftware.minlog:minlog</exclude>
<exclude>org.mortbay.jetty:*</exclude>
</excludes>
</artifactSet>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.sungard.dataflow.SymbolTransformPipeline</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>local</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2</version>
<configuration>
<executable>java</executable>
<arguments>
<argument>-classpath</argument>
<classpath />
<argument>com.sungard.dataflow.SymbolTransformPipeline</argument>
<argument>--runner=DirectPipelineRunner</argument>
<argument>--project=${gcp.project}</argument>
<argument>--stagingLocation=${gcp.stagingLocation}</argument>
<argument>--jobName=${df.jobName}</argument>
<argument>--dataflowJobFile=${df.jobFile}</argument>
<argument>--autoscalingAlgorithm=${df.scalingAlgorithm}</argument>
<argument>--workerMachineType=${df.workerMachineType}</argument>
<argument>--diskSizeGb=${df.workerDiskSize}</argument>
<argument>--numWorkers=${df.numWorkers}</argument>
<argument>--inputFilePath=${df.input}</argument>
<argument>--outputFilePath=${df.outputFile}</argument>
</arguments>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>gcp</id>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2</version>
<configuration>
<executable>java</executable>
<arguments>
<argument>-classpath</argument>
<classpath />
<argument>com.sungard.dataflow.SymbolTransformPipeline</argument>
<argument>--runner=BlockingDataflowPipelineRunner</argument>
<argument>--project=${gcp.project}</argument>
<argument>--stagingLocation=${gcp.stagingLocation}</argument>
<argument>--jobName=${df.jobName}</argument>
<argument>--dataflowJobFile=${df.jobFile}</argument>
<argument>--autoscalingAlgorithm=${df.scalingAlgorithm}</argument>
<argument>--workerMachineType=${df.workerMachineType}</argument>
<argument>--diskSizeGb=${df.workerDiskSize}</argument>
<argument>--numWorkers=${df.numWorkers}</argument>
<argument>--inputFilePath=${df.input}</argument>
<argument>--outputTable=${df.outputTable}</argument>
<!-- <argument>- -outputFilePath=${df.outputFile}</argument> -->
</arguments>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>