add table statistics

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java

@@ -32,6 +32,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.TreeMap;
 
 /** TSFileMetaData collects all metadata info and saves in its data structure. */
@@ -43,6 +44,7 @@ public class TsFileMetadata {
   // List of <name, offset, childMetadataIndexType>
   private Map<String, MetadataIndexNode> tableMetadataIndexNodeMap;
   private Map<String, TableSchema> tableSchemaMap;
+  private int tableSchemaNum;
   private boolean hasTableSchemaMapCache;
   private Map<String, String> tsFileProperties;
 
@@ -57,6 +59,8 @@ public class TsFileMetadata {
 
   private String encryptType;
 
+  private long tableStatisticsOffset = -1;
+
   public static TsFileMetadata deserializeAndCacheTableSchemaMap(
       ByteBuffer buffer, DeserializeConfig context) {
     return deserializeFrom(buffer, context, true);
@@ -90,9 +94,9 @@ public static TsFileMetadata deserializeFrom(
     fileMetaData.setTableMetadataIndexNodeMap(tableIndexNodeMap);
 
     // tableSchemas
-    int tableSchemaNum = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+    fileMetaData.tableSchemaNum = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
     Map<String, TableSchema> tableSchemaMap = new HashMap<>();
-    for (int i = 0; i < tableSchemaNum; i++) {
+    for (int i = 0; i < fileMetaData.tableSchemaNum; i++) {
       String tableName = ReadWriteIOUtils.readVarIntString(buffer);
       TableSchema tableSchema = context.tableSchemaBufferDeserializer.deserialize(buffer, context);
       if (needTableSchemaMap) {
@@ -122,6 +126,10 @@ public static TsFileMetadata deserializeFrom(
         String value = ReadWriteIOUtils.readVarIntString(buffer);
         propertiesMap.put(key, value);
       }
+      String tableStatisticsOffsetStr = propertiesMap.get("tableStatisticsOffset");
+      if (tableStatisticsOffsetStr != null) {
+        fileMetaData.tableStatisticsOffset = Long.parseLong(tableStatisticsOffsetStr);
+      }
       // if the file is not encrypted, set the default value(for compatible reason)
       if (!propertiesMap.containsKey("encryptLevel") || propertiesMap.get("encryptLevel") == null) {
         propertiesMap.put("encryptLevel", "0");
@@ -289,4 +297,12 @@ public Map<String, TableSchema> getTableSchemaMap() {
   public Map<String, String> getTsFileProperties() {
     return tsFileProperties;
   }
+
+  public Optional<Long> getTableStatisticsOffset() {
+    return tableStatisticsOffset > 0 ? Optional.of(tableStatisticsOffset) : Optional.empty();
+  }
+
+  public int getTableSchemaNum() {
+    return tableSchemaNum;
+  }
 }

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java

@@ -81,9 +81,9 @@ private void updateLastStats(Binary lastValue) {
 
   private void updateStats(Binary firstValue, Binary lastValue, long startTime, long endTime) {
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java

@@ -75,9 +75,9 @@ private void updateStats(boolean lastValue, long sum) {
   private void updateStats(
       boolean firstValue, boolean lastValue, long startTime, long endTime, long sum) {
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java

@@ -101,9 +101,9 @@ private void updateStats(
     }
     this.sumValue += sumValue;
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java

@@ -92,9 +92,9 @@ private void updateStats(
     }
     this.sumValue += sumValue;
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = first;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java

@@ -93,9 +93,9 @@ private void updateStats(
     }
     this.sumValue += sumValue;
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java

@@ -150,9 +150,9 @@ private void updateStats(
     }
     this.sumValue += sumValue;
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java

@@ -95,15 +95,15 @@ private void updateStats(
       long startTime,
       long endTime) {
     // only if endTime greater or equals to the current endTime need we update the last value
-    // only if startTime less or equals to the current startTime need we update the first value
+    // only if startTime less to the current startTime need we update the first value
     // otherwise, just ignore
     if (this.minValue.compareTo(minValue) > 0) {
       this.minValue = minValue;
     }
     if (this.maxValue.compareTo(maxValue) < 0) {
       this.maxValue = maxValue;
     }
-    if (startTime <= this.getStartTime()) {
+    if (startTime < this.getStartTime()) {
       this.firstValue = firstValue;
     }
     if (endTime >= this.getEndTime()) {

java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TableStatistics.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.file.metadata.statistics;
+
+import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class TableStatistics {
+  private final Map<String, Statistics<? extends Serializable>> fieldColumnStatisticsMap =
+      new TreeMap<>();
+
+  public void updateStatistics(
+      String fieldColumnName, Statistics<? extends Serializable> statistics) {
+    fieldColumnStatisticsMap
+        .computeIfAbsent(fieldColumnName, k -> Statistics.getStatsByType(statistics.getType()))
+        .mergeStatistics(statistics);
+  }
+
+  public int columnCount() {
+    return fieldColumnStatisticsMap.size();
+  }
+
+  public TimeStatistics getTimeStatistics() {
+    return (TimeStatistics) fieldColumnStatisticsMap.get(TsFileConstant.TIME_COLUMN_ID);
+  }
+
+  public Statistics<? extends Serializable> getStatistics(String fieldName) {
+    return fieldColumnStatisticsMap.get(fieldName);
+  }
+
+  public static TableStatistics deserialize(InputStream inputStream, Set<String> queriedColumns)
+      throws IOException {
+    TableStatistics tableStatistics = new TableStatistics();
+    List<String> columnNameList = ReadWriteIOUtils.readStringList(inputStream);
+    List<TSDataType> dataTypeList = new ArrayList<>(columnNameList.size());
+    List<Integer> statisticsSizeList = new ArrayList<>(columnNameList.size());
+    for (int i = 0; i < columnNameList.size(); i++) {
+      dataTypeList.add(ReadWriteIOUtils.readDataType(inputStream));
+    }
+    for (int i = 0; i < columnNameList.size(); i++) {
+      statisticsSizeList.add(ReadWriteForEncodingUtils.readVarInt(inputStream));
+    }
+
+    for (int i = 0; i < columnNameList.size(); i++) {
+      String columnName = columnNameList.get(i);
+      if (queriedColumns != null) {
+        if (tableStatistics.columnCount() >= queriedColumns.size()) {
+          break;
+        }
+        if (!queriedColumns.contains(columnName)) {
+          inputStream.skip(statisticsSizeList.get(i));
+          continue;
+        }
+      }
+      Statistics<? extends Serializable> columnStatistics =
+          Statistics.deserialize(inputStream, dataTypeList.get(i));
+      tableStatistics.updateStatistics(columnName, columnStatistics);
+    }
+    return tableStatistics;
+  }
+
+  public static TableStatistics deserialize(ByteBuffer byteBuffer, Set<String> queriedColumns)
+      throws IOException {
+    TableStatistics tableStatistics = new TableStatistics();
+    List<String> columnNameList = ReadWriteIOUtils.readStringList(byteBuffer);
+    List<TSDataType> dataTypeList = new ArrayList<>(columnNameList.size());
+    List<Integer> statisticsSizeList = new ArrayList<>(columnNameList.size());
+    for (int i = 0; i < columnNameList.size(); i++) {
+      dataTypeList.add(ReadWriteIOUtils.readDataType(byteBuffer));
+    }
+    for (int i = 0; i < columnNameList.size(); i++) {
+      statisticsSizeList.add(ReadWriteForEncodingUtils.readVarInt(byteBuffer));
+    }
+
+    for (int i = 0; i < columnNameList.size(); i++) {
+      String columnName = columnNameList.get(i);
+      if (queriedColumns != null) {
+        if (tableStatistics.columnCount() >= queriedColumns.size()) {
+          break;
+        }
+        if (!queriedColumns.contains(columnName)) {
+          byteBuffer.position(byteBuffer.position() + statisticsSizeList.get(i));
+          continue;
+        }
+      }
+      Statistics<? extends Serializable> columnStatistics =
+          Statistics.deserialize(byteBuffer, dataTypeList.get(i));
+      tableStatistics.updateStatistics(columnName, columnStatistics);
+    }
+    return tableStatistics;
+  }
+
+  public void serializeTo(OutputStream outputStream) throws IOException {
+    ReadWriteIOUtils.write(fieldColumnStatisticsMap.size(), outputStream);
+    for (String fieldName : fieldColumnStatisticsMap.keySet()) {
+      ReadWriteIOUtils.write(fieldName, outputStream);
+    }
+    for (Statistics<? extends Serializable> statistics : fieldColumnStatisticsMap.values()) {
+      ReadWriteIOUtils.write(statistics.getType(), outputStream);
+    }
+    for (Statistics<? extends Serializable> statistics : fieldColumnStatisticsMap.values()) {
+      ReadWriteForEncodingUtils.writeVarInt(statistics.getSerializedSize(), outputStream);
+    }
+    for (Statistics<? extends Serializable> statistics : fieldColumnStatisticsMap.values()) {
+      statistics.serialize(outputStream);
+    }
+  }
+}

java/tsfile/src/main/java/org/apache/tsfile/read/ITsFileTableStatisticsReader.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read;
+
+import org.apache.tsfile.file.metadata.statistics.TableStatistics;
+
+import java.io.IOException;
+import java.util.Map;
+
+public interface ITsFileTableStatisticsReader {
+  TableStatistics getTableStatistics(String tableName) throws IOException;
+
+  TableStatistics getTableFieldColumnStatistics(String tableName, String... fieldNames)
+      throws IOException;
+
+  Map<String, TableStatistics> getAllTableStatistics() throws IOException;
+}