diff --git a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java index 4850d1ade..11437f5d7 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/TsPrimitiveType.java @@ -22,6 +22,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Arrays; public abstract class TsPrimitiveType implements Serializable { @@ -36,8 +37,9 @@ public static TsPrimitiveType getByType(TSDataType dataType) { case BOOLEAN: return new TsPrimitiveType.TsBoolean(); case INT32: - case DATE: return new TsPrimitiveType.TsInt(); + case DATE: + return new TsPrimitiveType.TsInt(TSDataType.DATE); case INT64: case TIMESTAMP: return new TsPrimitiveType.TsLong(); @@ -68,8 +70,9 @@ public static TsPrimitiveType getByType(TSDataType dataType, Object v) { case BOOLEAN: return new TsPrimitiveType.TsBoolean((boolean) v); case INT32: - case DATE: return new TsPrimitiveType.TsInt((int) v); + case DATE: + return new TsPrimitiveType.TsInt((int) v, TSDataType.DATE); case INT64: case TIMESTAMP: return new TsPrimitiveType.TsLong((long) v); @@ -198,6 +201,11 @@ public void setBoolean(boolean val) { this.value = val; } + @Override + public Binary getBinary() { + return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); + } + @Override public void setObject(Object val) { if (val instanceof Boolean) { @@ -254,12 +262,23 @@ public static class TsInt extends TsPrimitiveType { private int value; + private TSDataType dataType = TSDataType.INT32; + public TsInt() {} public TsInt(int value) { this.value = value; } + public TsInt(TSDataType dataType) { + this.dataType = dataType; + } + + public TsInt(int value, TSDataType dataType) { + this.value = value; + this.dataType = dataType; + } + @Override public int getInt() { return value; @@ -280,6 +299,11 @@ public float getFloat() { return (float) value; } + @Override + public Binary getBinary() { + return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); + } + @Override public void setInt(int val) { this.value = val; @@ -316,7 +340,7 @@ public String getStringValue() { @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override @@ -357,6 +381,11 @@ public double getDouble() { return (double) value; } + @Override + public Binary getBinary() { + return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); + } + @Override public void setLong(long val) { this.value = val; @@ -434,6 +463,11 @@ public double getDouble() { return (double) value; } + @Override + public Binary getBinary() { + return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); + } + @Override public void setFloat(float val) { this.value = val; @@ -506,6 +540,11 @@ public double getDouble() { return value; } + @Override + public Binary getBinary() { + return new Binary(String.valueOf(this.value), StandardCharsets.UTF_8); + } + @Override public void setDouble(double val) { this.value = val; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java index e06f8c377..edd35e726 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/TsBlockBuilder.java @@ -111,10 +111,18 @@ private TsBlockBuilder(int initialExpectedEntries, int maxTsBlockBytes, List types) { tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); break; case INT32: + valueColumnBuilders[i] = + new IntColumnBuilder( + tsBlockBuilderStatus.createColumnBuilderStatus(), + initialExpectedEntries, + TSDataType.INT32); + break; case DATE: valueColumnBuilders[i] = new IntColumnBuilder( - tsBlockBuilderStatus.createColumnBuilderStatus(), initialExpectedEntries); + tsBlockBuilderStatus.createColumnBuilderStatus(), + initialExpectedEntries, + TSDataType.DATE); break; case INT64: case TIMESTAMP: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java index 7b9aca747..b00a5a66c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/BooleanColumn.java @@ -22,9 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -93,11 +95,25 @@ public boolean getBoolean(int position) { return values[position + arrayOffset]; } + @Override + public Binary getBinary(int position) { + return new Binary(String.valueOf(values[position + arrayOffset]), StandardCharsets.UTF_8); + } + @Override public boolean[] getBooleans() { return values; } + @Override + public Binary[] getBinaries() { + Binary[] binaries = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + binaries[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return binaries; + } + @Override public Object getObject(int position) { return getBoolean(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java index fc9774a15..4b5c468ae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java @@ -38,8 +38,9 @@ public static Column create(TSDataType dataType, int initialCapacity) { case OBJECT: return new BinaryColumn(initialCapacity); case INT32: + return new IntColumn(initialCapacity, TSDataType.INT32); case DATE: - return new IntColumn(initialCapacity); + return new IntColumn(initialCapacity, TSDataType.DATE); case BOOLEAN: return new BooleanColumn(initialCapacity); default: diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java index e0aff8f7a..7c99610c0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/DoubleColumn.java @@ -22,9 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -94,11 +96,25 @@ public double getDouble(int position) { return values[position + arrayOffset]; } + @Override + public Binary getBinary(int position) { + return new Binary(String.valueOf(values[position + arrayOffset]), StandardCharsets.UTF_8); + } + @Override public double[] getDoubles() { return values; } + @Override + public Binary[] getBinaries() { + Binary[] binaries = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + binaries[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return binaries; + } + @Override public Object getObject(int position) { return getDouble(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java index 8a576c0ce..85cd7916f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java @@ -22,9 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -100,6 +102,11 @@ public double getDouble(int position) { return values[position + arrayOffset]; } + @Override + public Binary getBinary(int position) { + return new Binary(String.valueOf(values[position + arrayOffset]), StandardCharsets.UTF_8); + } + @Override public float[] getFloats() { return values; @@ -114,6 +121,15 @@ public double[] getDoubles() { return doubles; } + @Override + public Binary[] getBinaries() { + Binary[] binaries = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + binaries[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return binaries; + } + @Override public Object getObject(int position) { return getFloat(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java index 6820a83eb..6e782b860 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java @@ -22,9 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -48,14 +50,27 @@ public class IntColumn implements Column { private final long retainedSizeInBytes; + private TSDataType dataType = TSDataType.INT32; + public IntColumn(int initialCapacity) { this(0, 0, null, new int[initialCapacity]); } + public IntColumn(int initialCapacity, TSDataType dataType) { + this(initialCapacity); + this.dataType = dataType; + } + public IntColumn(int positionCount, Optional valueIsNull, int[] values) { this(0, positionCount, valueIsNull.orElse(null), values); } + public IntColumn( + int positionCount, Optional valueIsNull, int[] values, TSDataType dataType) { + this(positionCount, valueIsNull, values); + this.dataType = dataType; + } + IntColumn(int arrayOffset, int positionCount, boolean[] valueIsNull, int[] values) { if (arrayOffset < 0) { throw new IllegalArgumentException("arrayOffset is negative"); @@ -80,6 +95,16 @@ public IntColumn(int positionCount, Optional valueIsNull, int[] value INSTANCE_SIZE + sizeOfIntArray(positionCount) + sizeOfBooleanArray(positionCount); } + IntColumn( + int arrayOffset, + int positionCount, + boolean[] valueIsNull, + int[] values, + TSDataType dataType) { + this(arrayOffset, positionCount, valueIsNull, values); + this.dataType = dataType; + } + @Override public TSDataType getDataType() { return TSDataType.INT32; @@ -110,6 +135,11 @@ public double getDouble(int position) { return values[position + arrayOffset]; } + @Override + public Binary getBinary(int position) { + return new Binary(String.valueOf(values[position + arrayOffset]), StandardCharsets.UTF_8); + } + @Override public int[] getInts() { return values; @@ -142,6 +172,15 @@ public double[] getDoubles() { return result; } + @Override + public Binary[] getBinaries() { + Binary[] binaries = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + binaries[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return binaries; + } + @Override public Object getObject(int position) { return getInt(position); @@ -149,7 +188,7 @@ public Object getObject(int position) { @Override public TsPrimitiveType getTsPrimitiveType(int position) { - return new TsPrimitiveType.TsInt(getInt(position)); + return new TsPrimitiveType.TsInt(getInt(position), dataType); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java index a0fc7a5b4..742e17767 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumnBuilder.java @@ -56,6 +56,8 @@ public class IntColumnBuilder implements ColumnBuilder { private long retainedSizeInBytes; + private TSDataType dataType = TSDataType.INT32; + public IntColumnBuilder(ColumnBuilderStatus columnBuilderStatus, int expectedEntries) { this.columnBuilderStatus = columnBuilderStatus; this.initialEntryCount = max(expectedEntries, 1); @@ -63,6 +65,12 @@ public IntColumnBuilder(ColumnBuilderStatus columnBuilderStatus, int expectedEnt updateDataSize(); } + public IntColumnBuilder( + ColumnBuilderStatus columnBuilderStatus, int expectedEntries, TSDataType dataType) { + this(columnBuilderStatus, expectedEntries); + this.dataType = dataType; + } + @Override public int getPositionCount() { return positionCount; @@ -128,12 +136,13 @@ public Column build() { if (!hasNonNullValue) { return new RunLengthEncodedColumn(NULL_VALUE_BLOCK, positionCount); } - return new IntColumn(0, positionCount, hasNullValue ? valueIsNull : null, values); + return new IntColumn( + 0, positionCount, hasNullValue ? valueIsNull : null, values, getDataType()); } @Override public TSDataType getDataType() { - return TSDataType.INT32; + return dataType; } @Override @@ -143,7 +152,8 @@ public long getRetainedSizeInBytes() { @Override public ColumnBuilder newColumnBuilderLike(ColumnBuilderStatus columnBuilderStatus) { - return new IntColumnBuilder(columnBuilderStatus, calculateBlockResetSize(positionCount)); + return new IntColumnBuilder( + columnBuilderStatus, calculateBlockResetSize(positionCount), dataType); } private void growCapacity() { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java index 03d8af0e6..b2dd5d1a4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java @@ -22,9 +22,11 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnEncoding; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.TsPrimitiveType; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Optional; @@ -100,6 +102,11 @@ public double getDouble(int position) { return values[position + arrayOffset]; } + @Override + public Binary getBinary(int position) { + return new Binary(String.valueOf(values[position + arrayOffset]), StandardCharsets.UTF_8); + } + @Override public long[] getLongs() { return values; @@ -114,6 +121,15 @@ public double[] getDoubles() { return doubles; } + @Override + public Binary[] getBinaries() { + Binary[] binaries = new Binary[values.length]; + for (int i = 0; i < values.length; i++) { + binaries[i] = new Binary(String.valueOf(values[i]), StandardCharsets.UTF_8); + } + return binaries; + } + @Override public Object getObject(int position) { return getLong(position); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java index ff34c98b2..50835f5f8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/reader/page/ValuePageReader.java @@ -190,7 +190,10 @@ public TsPrimitiveType nextValue(long timestamp, int timeIndex) throws IOExcepti case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timestamp)) { - resultValue = new TsPrimitiveType.TsInt(anInt); + resultValue = + dataType.equals(TSDataType.INT32) + ? new TsPrimitiveType.TsInt(anInt) + : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); } break; case INT64: @@ -253,7 +256,10 @@ public TsPrimitiveType[] nextValueBatch(long[] timeBatch) throws IOException { case DATE: int anInt = valueDecoder.readInt(valueBuffer); if (!isDeleted(timeBatch[i])) { - valueBatch[i] = new TsPrimitiveType.TsInt(anInt); + valueBatch[i] = + dataType.equals(TSDataType.INT32) + ? new TsPrimitiveType.TsInt(anInt) + : new TsPrimitiveType.TsInt(anInt, TSDataType.DATE); } break; case INT64: diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java index 0cb3f2e78..f3a784324 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/common/ColumnTest.java @@ -38,11 +38,13 @@ import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; import org.apache.tsfile.read.common.block.column.TimeColumn; import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BytesUtils; import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.Optional; public class ColumnTest { @@ -224,7 +226,11 @@ public void booleanColumnFilterPositionsTest() { Assert.assertTrue(columnByGetPositions instanceof DictionaryColumn); Assert.assertEquals(2, columnByGetPositions.getPositionCount()); Assert.assertFalse(columnByGetPositions.getBoolean(0)); + Assert.assertEquals( + new Binary("false", StandardCharsets.UTF_8), columnByGetPositions.getBinary(0)); Assert.assertTrue(columnByGetPositions.getBoolean(1)); + Assert.assertEquals( + new Binary("true", StandardCharsets.UTF_8), columnByGetPositions.getBinary(1)); Column columnByCopyPositions = originalColumn.copyPositions(selectedPositions, 1, 2); Assert.assertEquals(2, columnByCopyPositions.getPositionCount()); @@ -242,7 +248,9 @@ public void doubleColumnSubColumnTest() { doubleColumn1 = (DoubleColumn) doubleColumn1.subColumn(5); Assert.assertEquals(5, doubleColumn1.getPositionCount()); Assert.assertEquals(5.0, doubleColumn1.getDouble(0), 0.001); + Assert.assertEquals(new Binary("5.0", StandardCharsets.UTF_8), doubleColumn1.getBinary(0)); Assert.assertEquals(9.0, doubleColumn1.getDouble(4), 0.001); + Assert.assertEquals(new Binary("9.0", StandardCharsets.UTF_8), doubleColumn1.getBinary(4)); DoubleColumn doubleColumn2 = (DoubleColumn) doubleColumn1.subColumn(3); Assert.assertEquals(2, doubleColumn2.getPositionCount()); @@ -303,7 +311,9 @@ public void floatColumnSubColumnTest() { floatColumn1 = (FloatColumn) floatColumn1.subColumn(5); Assert.assertEquals(5, floatColumn1.getPositionCount()); Assert.assertEquals(5.0, floatColumn1.getFloat(0), 0.001); + Assert.assertEquals(new Binary("5.0", StandardCharsets.UTF_8), floatColumn1.getBinary(0)); Assert.assertEquals(9.0, floatColumn1.getFloat(4), 0.001); + Assert.assertEquals(new Binary("9.0", StandardCharsets.UTF_8), floatColumn1.getBinary(4)); FloatColumn floatColumn2 = (FloatColumn) floatColumn1.subColumn(3); Assert.assertEquals(2, floatColumn2.getPositionCount()); @@ -414,6 +424,10 @@ public void intColumnFilterPositionsTest() { Assert.assertEquals(2, columnByCopyPositions.getPositionCount()); Assert.assertEquals(3, columnByCopyPositions.getInt(0)); Assert.assertEquals(5, columnByCopyPositions.getInt(1)); + Assert.assertEquals( + new Binary("3", StandardCharsets.UTF_8), columnByCopyPositions.getBinary(0)); + Assert.assertEquals( + new Binary("5", StandardCharsets.UTF_8), columnByCopyPositions.getBinary(1)); } @Test @@ -427,11 +441,15 @@ public void longColumnSubColumnTest() { Assert.assertEquals(5, longColumn1.getPositionCount()); Assert.assertEquals(5, longColumn1.getLong(0)); Assert.assertEquals(9, longColumn1.getLong(4)); + Assert.assertEquals(new Binary("5", StandardCharsets.UTF_8), longColumn1.getBinary(0)); + Assert.assertEquals(new Binary("9", StandardCharsets.UTF_8), longColumn1.getBinary(4)); LongColumn longColumn2 = (LongColumn) longColumn1.subColumn(3); Assert.assertEquals(2, longColumn2.getPositionCount()); Assert.assertEquals(8, longColumn2.getLong(0)); Assert.assertEquals(9, longColumn2.getLong(1)); + Assert.assertEquals(new Binary("8", StandardCharsets.UTF_8), longColumn2.getBinary(0)); + Assert.assertEquals(new Binary("9", StandardCharsets.UTF_8), longColumn2.getBinary(1)); Assert.assertSame(longColumn1.getLongs(), longColumn2.getLongs()); }