GTFS dataset files get zipped and validated

.gitignore

@@ -13,6 +13,9 @@ build/
 .settings
 .springBeans
 .sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
 
 ### IntelliJ IDEA ###
 .idea
@@ -33,8 +36,8 @@ out/
 ### VS Code ###
 .vscode/
 
-### Eclipse ###
-.classpath
-.project
-.settings/
-bin/
+### project-specific ###
+
+/data*/
+/src/main/resources/public/
+/tmp/

README.md

@@ -8,14 +8,14 @@ Define database connection and GTFS option properties in the **application.prope
 
 * spring.datasource.url
 * spring.datasource.username
-* spring.datasource.password
+* spring.datasource.password      
 ...
 
 or use environment variables:
 
 * DB_URL
 * DB_USERNAME
-* DB_PASSWORD
+* DB_PASSWORD      
 ...
 
 `./gradlew clean bootRun --args 'runOnce'`
@@ -26,4 +26,4 @@ to start the application and generate the GTFS once.
 
 to start the application and generate the GTFS by the given cron schedule.
 
-Default GTFS output directory is `data/`.
+Default GTFS output directory is `data/output/`. When using scheduling the GTFS file will also be published to http://localhost:8080/gtfs.zip

build.gradle

@@ -1,6 +1,6 @@
 plugins {
-  id 'org.springframework.boot' version '2.4.0'
-  id 'io.spring.dependency-management' version '1.0.10.RELEASE'
+  id 'org.springframework.boot' version '2.4.2'
+  id 'io.spring.dependency-management' version '1.0.11.RELEASE'
   id 'java'
   id 'groovy'
 }
@@ -14,13 +14,19 @@ configurations {
 }
 repositories {
   maven { url 'https://repo.spring.io/milestone' }
+  maven { url 'https://repo.osgeo.org/repository/release/' }
   mavenCentral()
+  maven {
+    url = 'http://nexus.onebusaway.org/content/repositories/releases/'
+    allowInsecureProtocol = true
+  }
 }
 ext {
-  set('springCloudVersion', "2020.0.0-M5")
+  set('springCloudVersion', "2020.0.1")
 }
 dependencies {
   implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+  implementation 'org.springframework.boot:spring-boot-starter-mail'
   implementation 'org.springframework.boot:spring-boot-starter-webflux'
   implementation 'org.springframework.cloud:spring-cloud-starter-circuitbreaker-reactor-resilience4j'
   // routing with GraphHopper
@@ -30,20 +36,28 @@ dependencies {
   // for XML WebClient response in OpenStreetMap node service
   implementation 'javax.xml.bind:jaxb-api:2.3.1'
   implementation 'javax.activation:activation:1.1.1'
-  implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.3'
+  implementation 'org.glassfish.jaxb:jaxb-runtime:2.3.1'
   // write GTFS
-  implementation 'org.onebusaway:onebusaway-gtfs:1.3.4'
-  // delete directory
+  implementation 'org.onebusaway:onebusaway-gtfs:1.3.87'
+  // GTFS validation
+  implementation ('com.conveyal:gtfs-validation-lib:0.1.8') {
+	exclude group: 'org.slf4j', module: 'slf4j-simple'
+  }
+  implementation ('com.conveyal:gtfs-validator-json:0.1.8') {
+	exclude group: 'org.slf4j', module: 'slf4j-simple'
+  }
+  // for util handling
   implementation 'commons-io:commons-io:2.8.0'
+  implementation 'org.apache.commons:commons-lang3:3.11'
 
   compileOnly 'org.projectlombok:lombok'
   runtimeOnly 'mysql:mysql-connector-java'
   annotationProcessor 'org.projectlombok:lombok'
   testImplementation 'org.springframework.boot:spring-boot-starter-test'
   testImplementation 'io.projectreactor:reactor-test'
-  testImplementation 'org.codehaus.groovy:groovy:3.0.6'
+  testImplementation 'org.codehaus.groovy:groovy:3.0.7'
   testImplementation 'org.codehaus.groovy:groovy-all:3.0.6'
-  testImplementation 'org.spockframework:spock-spring:2.0-M4-groovy-3.0'  
+  testImplementation 'org.spockframework:spock-spring:2.0-M4-groovy-3.0'
 }
 dependencyManagement {
   imports {

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -130,7 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-    
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath

gradlew.bat

@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -54,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -64,29 +64,14 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

src/main/java/com/ride2go/ridetogtfsconverter/RunService.java

@@ -2,18 +2,15 @@
 
 import static com.ride2go.ridetogtfsconverter.configuration.SystemConfiguration.AMOUNT_OF_THREADS;
 import static com.ride2go.ridetogtfsconverter.util.DateAndTimeHandler.TIME_ZONE_ID_BERLIN;
-import static com.ride2go.ridetogtfsconverter.validation.Constraints.AREA_BADEN_WUERTTEMBERG;
 
 import java.io.File;
-import java.io.IOException;
 import java.time.LocalDate;
 import java.util.List;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 
-import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -24,92 +21,96 @@
 import org.springframework.data.domain.Sort.Order;
 import org.springframework.stereotype.Service;
 
+import com.ride2go.ridetogtfsconverter.exception.OBAException;
 import com.ride2go.ridetogtfsconverter.gtfs.OBAWriterParameter;
 import com.ride2go.ridetogtfsconverter.gtfs.WriterService;
 import com.ride2go.ridetogtfsconverter.model.item.Offer;
+import com.ride2go.ridetogtfsconverter.normalization.ApplicationPropertiesNormalizer;
 import com.ride2go.ridetogtfsconverter.ridesdata.ReaderService;
 import com.ride2go.ridetogtfsconverter.routing.RoutingHandler;
 import com.ride2go.ridetogtfsconverter.util.DateAndTimeHandler;
+import com.ride2go.ridetogtfsconverter.util.FileHandler;
+import com.ride2go.ridetogtfsconverter.validation.ApplicationPropertiesValidator;
+import com.ride2go.ridetogtfsconverter.validation.GtfsValidator;
 
 @Service
 public class RunService {
 
 	private static final Logger LOG = LoggerFactory.getLogger(RunService.class);
 
-	private static final String DEFAULT_GTFS_OUTPUT_DIRECTORY = "data/";
+	private static final String PUBLIC_GTFS_FILE_LOCATION = "src/main/resources/public/gtfs.zip";
 
 	private static final int PAGE_SIZE = 5000;
 
 	private static final Sort TRIP_SORT = Sort.by(Order.desc("tripId"));
 
 	@Autowired
-	private WriterService writerService;
+	private ApplicationPropertiesNormalizer applicationPropertiesNormalizer;
+
+	@Autowired
+	private ApplicationPropertiesValidator applicationPropertiesValidator;
 
 	@Autowired
 	private ReaderService readerService;
 
 	@Autowired
 	private RoutingHandler routingHandler;
 
-	@Value("${custom.gtfs.output.directory}")
-	private String gtfsOutputDirectory;
+	@Autowired
+	private WriterService writerService;
 
-	@Value("${custom.trips.by-user}")
-	private String tripsByUser;
+	@Autowired
+	private GtfsValidator gtfsValidator;
+
+	@Value("${custom.gtfs.dataset.directory:data/dataset/}")
+	private String gtfsDatasetDirectory;
+
+	@Value("${custom.gtfs.file:data/output/gtfs.zip}")
+	private String gtfsFile;
 
-	@Value("${custom.gtfs.trips.area}")
-	private String area;
+	@Value("${custom.gtfs.validation.file:data/validation/results.json}")
+	private String gtfsValidationFile;
+
+	@Value("${custom.trips.by-user:#{null}}")
+	private String tripsByUser;
 
 	protected void run() throws Exception {
+		updateDateAndTimeParameter();
+		if (applicationPropertiesValidator.validDirectoriesAndFiles(
+				gtfsDatasetDirectory, gtfsFile, gtfsValidationFile)) {
+			applicationPropertiesValidator.validArea();
+			final File directory = new File(gtfsDatasetDirectory);
+			try {
+				writerService.writeProviderInfoAsGTFS(directory);
+				if (tripsByUser != null && !tripsByUser.trim().isEmpty()) {
+					LOG.info("Get all trips from User " + tripsByUser);
+					processTripsByUser(directory, tripsByUser);
+				} else {
+					LOG.info("UserId not specified. Get trips from all users.");
+					processAllTrips(directory);
+				}
+			} catch (OBAException e) {
+				LOG.error("Problem while writing GTFS data with OneBusAway: " + e.getMessage());
+			}
+			FileHandler.zipGtfsDatasetFiles(directory, gtfsFile);
+			gtfsValidator.setRecipients(
+					applicationPropertiesNormalizer.normalizeMailRecipientAddressArray(
+							gtfsValidator.getRecipients()));
+			applicationPropertiesValidator.validMailAddresses(
+					gtfsValidator.getRecipients());
+			if (gtfsValidator.check(gtfsFile, gtfsValidationFile)) {
+				FileHandler.exposeGtfsFile(gtfsFile, PUBLIC_GTFS_FILE_LOCATION);
+			}
+		}
+	}
+
+	private void updateDateAndTimeParameter() {
 		DateAndTimeHandler.today = LocalDate.now(TIME_ZONE_ID_BERLIN);
 		DateAndTimeHandler.oneMonthFromToday = DateAndTimeHandler.today.plusMonths(1);
 		DateAndTimeHandler.oneYearFromToday = DateAndTimeHandler.today.plusYears(1);
 		OBAWriterParameter.feedStartDate = DateAndTimeHandler.today;
 		OBAWriterParameter.feedEndDate = DateAndTimeHandler.oneMonthFromToday;
 		OBAWriterParameter.feedTimePeriodWeekDays = OBAWriterParameter.getFeedTimePeriodWeekDays();
-
-		final File directory = getDirectory();
-		LOG.info("Use directory " + directory);
-		final String userId = getUserId();
-		area();
-		writerService.writeProviderInfoAsGTFS(directory);
-		if (userId != null) {
-			LOG.info("Get all trips from User " + userId);
-			processTripsByUser(directory, userId);
-		} else {
-			LOG.info("UserId not specified. Get trips from all users.");
-			processAllTrips(directory);
-		}
-	}
-
-	private File getDirectory() throws IOException {
-		final String directoryString = getValueOrDefault(gtfsOutputDirectory, DEFAULT_GTFS_OUTPUT_DIRECTORY);
-		final File directory = new File(directoryString);
-		try {
-			if (directory.exists() && directory.isDirectory()) {
-				FileUtils.deleteDirectory(directory);
-			}
-		} catch (IOException e) {
-			LOG.error("Problem with given directory");
-			throw e;
-		}
-		return directory;
-	}
-
-	private String getUserId() {
-		return getValueOrDefault(tripsByUser, null);
-	}
-
-	private void area() {
-		if (area.isEmpty()) {
-			LOG.info("No area restriction");
-		} else {
-			if (area.equals(AREA_BADEN_WUERTTEMBERG)) {
-				LOG.info("Trips only from the area: " + area);
-			} else {
-				LOG.info("Area {} not recognized", area);
-			}
-		}
 	}
 
 	private void processTripsByUser(final File directory, String userId) {
@@ -141,14 +142,4 @@ private void processAllTrips(final File directory)
 			index += AMOUNT_OF_THREADS;
 		}
 	}
-
-	private String getValueOrDefault(String value, String defaultValue) {
-		if (value != null) {
-			value = value.trim();
-			if (!value.isEmpty()) {
-				return value;
-			}
-		}
-		return defaultValue;
-	}
 }

src/main/java/com/ride2go/ridetogtfsconverter/ScheduledService.java

@@ -20,15 +20,15 @@ public class ScheduledService {
 
 	private boolean finished = true;
 
-	@Scheduled(cron = "${custom.scheduling.job.cron}", zone = TIME_ZONE_BERLIN)
+	@Scheduled(cron = "${custom.scheduling.job.cron:}", zone = TIME_ZONE_BERLIN)
 	public void run() throws Exception {
 		if (enable) {
 			if (finished) {
 				finished = false;
 				runService.run();
 				finished = true;
 			} else {
-				LOG.info("Can't do a new run if the last run is not finished");
+				LOG.warn("Can't do a new run if the last run is not finished");
 			}
 		}
 	}

src/main/java/com/ride2go/ridetogtfsconverter/configuration/SystemConfiguration.java

@@ -67,7 +67,7 @@ public Customizer<ReactiveResilience4JCircuitBreakerFactory> defaultCustomizer()
 
 	@Bean
 	@Qualifier("configured")
-	public RoutingService getRoutingService(@Value("${custom.routing.service}") final String routingServiceChoice) {
+	public RoutingService getRoutingService(@Value("${custom.routing.service:}") final String routingServiceChoice) {
 		if (routingServiceChoice.equals("GH")) {
 			LOG.info("Use GraphHopper as routing engine");
 			return new GHRoutingService();