refactoring table; update java dependencies

.settings/org.eclipse.jdt.core.prefs

@@ -22,6 +22,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled
 org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled
 org.eclipse.jdt.core.compiler.problem.discouragedReference=warning
 org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
 org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
 org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore
 org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore
@@ -67,6 +68,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno
 org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore
 org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore
 org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
 org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled
 org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning
 org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled

build.gradle

@@ -2,7 +2,7 @@
 
 plugins {
   id 'org.kordamp.gradle.markdown' version '2.2.0' // https://github.com/aalmiray/markdown-gradle-plugin
-  id "com.github.ben-manes.versions" version '0.42.0' // update check task: dependencyUpdates  https://github.com/ben-manes/gradle-versions-plugin
+  id "com.github.ben-manes.versions" version '0.44.0' // update check task: dependencyUpdates  https://github.com/ben-manes/gradle-versions-plugin
 }
 
 apply plugin: 'java'
@@ -64,10 +64,10 @@ dependencies {
   	implementation group: 'org.json', name: 'json', version: '20220924'
   	implementation group: 'org.yaml', name: 'snakeyaml', version: '1.33'
   	//implementation group: 'com.opencsv', name: 'opencsv', version: '4.6' // warning: API at v5.0 breaks
-  	implementation group: 'com.opencsv', name: 'opencsv', version: '5.7.0'   	
+  	implementation group: 'com.opencsv', name: 'opencsv', version: '5.7.1'   	
   	implementation group: 'org.ini4j', name: 'ini4j', version: '0.5.4' 
   	implementation group: 'ar.com.hjg', name: 'pngj', version: '2.1.0'
-  	implementation group: 'com.github.luben', name: 'zstd-jni', version: '1.5.2-4' // caution: newer versions may break storage layout (last tested 1.4.5-6)
+  	implementation group: 'com.github.luben', name: 'zstd-jni', version: '1.5.2-5' // caution: newer versions may break storage layout (last tested 1.4.5-6)
   	implementation group: 'me.lemire.integercompression', name: 'JavaFastPFOR', version: '0.1.12' // caution: newer versions may break strorage layout
   	implementation group: 'ch.randelshofer', name: 'fastdoubleparser', version: '0.3.0'
 

src/tsdb/TsDB.java

@@ -33,7 +33,7 @@
  */
 public class TsDB implements AutoCloseable {
 
-	public static final String tubedb_version = "1.22.8";
+	public static final String tubedb_version = "1.22.9";
 
 	/**
 	 * map regionName -> Region

src/tsdb/loader/csv/ImportGenericCSV.java

@@ -82,7 +82,7 @@ public void loadFiles(Path rootPath) {
 	public void loadFile(Path filePath) {
 		try {
 			Logger.info("load file "+filePath);			
-			StreamTable table = StreamTable.readCSV(filePath,',');
+			StreamTable table = StreamTable.openCSV(filePath,',');
 
 			int stationIndex = getStationIndex(table);
 			if(stationIndex > 0) {

src/tsdb/loader/treetalker/Loader_TreeTalker.java

@@ -4,6 +4,7 @@
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -31,9 +32,7 @@
 import tsdb.util.TimeUtil;
 
 public class Loader_TreeTalker {
-
 
-	private static final Charset UTF8 = Charset.forName("UTF-8");
 	private static final char SEPARATOR = ';';
 	private static final LocalDateTime UNIX_EPOCH = LocalDateTime.of(1970,1,1,0,0);
 	private static final int UNIX_EPOCH_OLE_AUTOMATION_TIME_DIFFERENCE_MINUTES = (int) Duration.between(TimeUtil.OLE_AUTOMATION_TIME_START, UNIX_EPOCH).toMinutes();
@@ -93,7 +92,7 @@ static int toTimestamp(String timestampText) {
 
 	static class TreeTalkerTable {
 		public TreeTalkerTable(TsDB tsdb, File file) throws Exception {
-			InputStreamReader in = new InputStreamReader(new FileInputStream(file),UTF8);
+			InputStreamReader in = new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
 			//try(CSVReader reader = new CSVReader(in, SEPARATOR)) {
 			CSVParser csvParser = new CSVParserBuilder().withSeparator(SEPARATOR).build();
 			try(CSVReader reader = new CSVReaderBuilder(in).withCSVParser(csvParser).build()) {		

src/tsdb/remote/ZipExport.java

@@ -7,8 +7,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.rmi.RemoteException;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -26,7 +25,6 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
-
 import org.tinylog.Logger;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml;
@@ -49,8 +47,6 @@
  */
 public class ZipExport extends TimestampSeriesCSVwriter{
 
-	private static final Charset charset = Charset.forName("UTF-8");
-
 	private final RemoteTsDB tsdb;
 
 	private Consumer<String> cbPrintLine = null;
@@ -188,7 +184,7 @@ public boolean writeToStream(OutputStream outputstream) {
 
 			if(desc_settings) {
 				zipOutputStream.putNextEntry(new ZipEntry("processing_settings.yaml"));
-				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 				BufferedWriter bufferedWriter = new BufferedWriter(writer);
 				write_settings_YAML(bufferedWriter);
 				bufferedWriter.flush();
@@ -197,7 +193,7 @@ public boolean writeToStream(OutputStream outputstream) {
 
 			if(desc_sensor) {
 				zipOutputStream.putNextEntry(new ZipEntry("sensor_description.csv"));
-				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 				BufferedWriter bufferedWriter = new BufferedWriter(writer);
 				write_sensor_description_CSV(bufferedWriter);
 				bufferedWriter.flush();
@@ -206,7 +202,7 @@ public boolean writeToStream(OutputStream outputstream) {
 
 			if(desc_plot) {
 				zipOutputStream.putNextEntry(new ZipEntry("plot_description.csv"));
-				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 				BufferedWriter bufferedWriter = new BufferedWriter(writer);
 				write_plot_description_CSV(bufferedWriter);
 				bufferedWriter.flush();
@@ -216,7 +212,7 @@ public boolean writeToStream(OutputStream outputstream) {
 			if(info) {
 				if(region.description != null && !region.description.isEmpty()) {
 					zipOutputStream.putNextEntry(new ZipEntry("info.txt"));
-					OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+					OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 					BufferedWriter bufferedWriter = new BufferedWriter(writer);
 					write_info(bufferedWriter);
 					bufferedWriter.flush();
@@ -227,7 +223,7 @@ public boolean writeToStream(OutputStream outputstream) {
 			if(plots_aggregate) {
 				printLine("processing plots_aggregate ...");
 				zipOutputStream.putNextEntry(new ZipEntry("plots_aggregated.csv"));
-				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+				OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 				BufferedWriter bufferedWriter = new BufferedWriter(writer);
 				if(write_header) {
 					writeCSVHeader(bufferedWriter, sensorNames, false);
@@ -254,7 +250,7 @@ public boolean writeToStream(OutputStream outputstream) {
 					if(casted) {
 						printLine("processing plots_casted ...");
 						zipOutputStream.putNextEntry(new ZipEntry("plots.csv"));
-						OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+						OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 						BufferedWriter bufferedWriter = new BufferedWriter(writer);
 						try {
 							TimestampSeries timeseries = tsdb.plots_casted(plotIDs, sensorNames, aggregationInterval, dataQuality, interpolated, startTimestamp, endTimestamp);
@@ -278,7 +274,7 @@ public boolean writeToStream(OutputStream outputstream) {
 						writer.flush();						
 					} else {
 						zipOutputStream.putNextEntry(new ZipEntry("plots.csv"));
-						OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+						OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 						BufferedWriter bufferedWriter = new BufferedWriter(writer);
 						if(write_header) {
 							writeCSVHeader(bufferedWriter, sensorNames, col_plotid);
@@ -316,7 +312,7 @@ public boolean writeToStream(OutputStream outputstream) {
 								TimestampSeries timeseries = tsdb.plot(null,plotID, schema, aggregationInterval, dataQuality, interpolated, startTimestamp, endTimestamp);
 								if(timeseries!=null) {
 									zipOutputStream.putNextEntry(new ZipEntry(plotID+".csv"));
-									OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, charset);
+									OutputStreamWriter writer = new OutputStreamWriter(zipOutputStream, StandardCharsets.UTF_8);
 									BufferedWriter bufferedWriter = new BufferedWriter(writer);
 									if(write_header) {
 										writeCSVHeader(bufferedWriter, sensorNames, col_plotid);

src/tsdb/usecase/TestingCharset.java

@@ -1,6 +1,7 @@
 package tsdb.usecase;
 
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.SortedMap;
 
 public class TestingCharset {
@@ -13,9 +14,9 @@ public static void main(String[] args) {
 			System.out.println(k);
 		}
 		System.out.println();
-		System.out.println("default: "+Charset.defaultCharset());
-		System.out.println("UTF-8: "+Charset.forName("UTF-8"));
-		System.out.println("windows-1252: "+Charset.forName("windows-1252"));
+		System.out.println("default: " + Charset.defaultCharset());
+		System.out.println("UTF-8: " + StandardCharsets.UTF_8);
+		System.out.println("windows-1252: " + Charset.forName("windows-1252"));
 	}
 
 }

src/tsdb/util/StreamTable.java

@@ -7,6 +7,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
 import org.tinylog.Logger;
@@ -21,7 +22,7 @@ public static StreamTable openCSV(String filename, char separator) throws FileNo
 		return openCSV(new File(filename), separator);
 	}
 
-	public static StreamTable readCSV(Path filename, char separator) throws FileNotFoundException, IOException {
+	public static StreamTable openCSV(Path filename, char separator) throws FileNotFoundException, IOException {
 		return openCSV(filename.toFile(),separator);
 	}
 
@@ -38,14 +39,14 @@ public static StreamTable openCSV(File file, char separator) throws FileNotFound
 	}
 
 	public static StreamTable openCSV(InputStream in, char separator) throws IOException {
-		Reader reader = new InputStreamReader(in, Table.UTF8);
+		Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8);
 		return openCSV(reader, separator);	
 	}
 
 	public static StreamTable openCSV(Reader reader, char separator) throws IOException {
 		StreamTable table = new StreamTable();
-		CSVReader csvReader = Table.buildCSVReader(reader, separator);
-		if(Table.readHeader(table, csvReader)) {
+		CSVReader csvReader = TableUtil.buildCSVReader(reader, separator);
+		if(TableUtil.readHeader(table, csvReader)) {
 			table.csvReader = csvReader;
 			return table;
 		} else {

src/tsdb/util/Table.java

@@ -7,7 +7,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 
@@ -25,9 +25,6 @@
  */
 public class Table extends AbstractTable {
 
-	static final Charset UTF8 = Charset.forName("UTF-8");
-	private static final String UTF8_BOM = "\uFEFF";
-
 	/**
 	 * table rows of csv file
 	 */
@@ -58,7 +55,7 @@ public static Table readCSV(File file, char separator) {
 	}
 
 	public static Table readCSV(InputStream in, char separator) {
-		try(InputStreamReader reader = new InputStreamReader(in, UTF8)) {
+		try(InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) {
 			return readCSV(reader, separator);			
 		} catch(Exception e) {
 			Logger.error(e);
@@ -69,9 +66,9 @@ public static Table readCSV(InputStream in, char separator) {
 	public static Table readCSV(Reader reader, char separator) {
 		try {
 			Table table = new Table();
-			try(CSVReader csvReader = buildCSVReader(reader, separator)) {
+			try(CSVReader csvReader = TableUtil.buildCSVReader(reader, separator)) {
 				//List<String[]> list = reader.readAll(); // very slow because of linkedlist for indexed access
-				if(Table.readHeader(table, csvReader)) {
+				if(TableUtil.readHeader(table, csvReader)) {
 					table.readRows(csvReader);
 				}
 			}
@@ -84,9 +81,9 @@ public static Table readCSV(Reader reader, char separator) {
 
 	public static Table readCSVThrow(Reader reader, char separator) throws Exception {
 		Table table = new Table();
-		try(CSVReader csvReader = buildCSVReader(reader, separator)) {
+		try(CSVReader csvReader = TableUtil.buildCSVReader(reader, separator)) {
 			//List<String[]> list = reader.readAll(); // very slow because of linkedlist for indexed access
-			if(Table.readHeader(table, csvReader)) {
+			if(TableUtil.readHeader(table, csvReader)) {
 				table.readRows(csvReader);
 			}
 		}
@@ -105,8 +102,8 @@ public static Table readCSVFirstDataRow(String filename, char separator) {
 	public static Table readCSVFirstDataRow(Reader reader, char separator) {
 		try {
 			Table table = new Table();
-			try(CSVReader csvReader = buildCSVReader(reader, separator)) {
-				if(Table.readHeader(table, csvReader)) {
+			try(CSVReader csvReader = TableUtil.buildCSVReader(reader, separator)) {
+				if(TableUtil.readHeader(table, csvReader)) {
 					String[] dataRow = csvReader.readNext();
 					if(dataRow != null) {
 						table.rows = new String[][] {dataRow};
@@ -123,28 +120,6 @@ public static Table readCSVFirstDataRow(Reader reader, char separator) {
 			return null;
 		}
 	}
-
-	static CSVReader buildCSVReader(Reader reader, char separator) {
-		CSVParser csvParser = new CSVParserBuilder().withSeparator(separator).build();
-		return new CSVReaderBuilder(reader).withCSVParser(csvParser).build();
-	}
-
-	static boolean readHeader(AbstractTable table, CSVReader csvReader) throws IOException {
-		String[] curRow = csvReader.readNextSilently();
-		if(curRow != null) {
-			String[] columnsNames = curRow;
-			if(columnsNames.length>0) { // filter UTF8 BOM
-				if(columnsNames[0].startsWith(UTF8_BOM)) {
-					columnsNames[0] = columnsNames[0].substring(1, columnsNames[0].length());
-				}
-			}			
-			table.updateNames(columnsNames);
-			//Logger.info("names: "+Arrays.toString(table.names)+"   in "+filename);
-			return true;
-		} else {
-			return false;
-		}
-	}
 
 	private void readRows(CSVReader reader) throws IOException {
 		ArrayList<String[]> dataRowList = readRowList(reader);				

src/tsdb/util/TableUtil.java

@@ -0,0 +1,36 @@
+package tsdb.util;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+
+public class TableUtil {
+
+	private static final String UTF8_BOM = "\uFEFF";
+
+	static CSVReader buildCSVReader(Reader reader, char separator) {
+		CSVParser csvParser = new CSVParserBuilder().withSeparator(separator).build();
+		return new CSVReaderBuilder(reader).withCSVParser(csvParser).build();
+	}
+
+	static boolean readHeader(AbstractTable table, CSVReader csvReader) throws IOException {
+		String[] curRow = csvReader.readNextSilently();
+		if(curRow != null) {
+			String[] columnsNames = curRow;
+			if(columnsNames.length>0) { // filter UTF8 BOM
+				if(columnsNames[0].startsWith(UTF8_BOM)) {
+					columnsNames[0] = columnsNames[0].substring(1, columnsNames[0].length());
+				}
+			}			
+			table.updateNames(columnsNames);
+			//Logger.info("names: "+Arrays.toString(table.names)+"   in "+filename);
+			return true;
+		} else {
+			return false;
+		}
+	}
+}

src/tsdb/util/Util.java

@@ -5,6 +5,7 @@
 import java.io.File;
 import java.lang.reflect.Array;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.util.ArrayList;
@@ -39,7 +40,6 @@
  *
  */
 public final class Util {
-	public static final Charset CHARSET_UTF_8 = Charset.forName("UTF-8");
 
 	private Util(){}
 
@@ -772,7 +772,7 @@ public static String removeComments(byte[] data) {
 		if(!inComment && i<=size_max) {
 			result[r++] = data[i++];
 		}
-		return new String(result, 0, r, CHARSET_UTF_8);
+		return new String(result, 0, r, StandardCharsets.UTF_8);
 	}
 
 	/**