diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 693b4ffb1..7f043c076 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -33,10 +33,12 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class MetadataIndexNode { + private static final double LOG2 = Math.log(2); protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); protected final List children; protected long endOffset; @@ -127,6 +129,10 @@ public static MetadataIndexNode deserializeFrom( public Pair getChildIndexEntry(Comparable key, boolean exactSearch) { int index = binarySearchInChildren(key, exactSearch); + return getChildIndexEntry(index); + } + + private Pair getChildIndexEntry(int index) { if (index == -1) { return null; } @@ -165,6 +171,105 @@ int binarySearchInChildren(Comparable key, boolean exactSearch) { } } + public List> getChildIndexEntries( + List keys, boolean exactSearch) { + int[] indexArr = + keys.size() >= children.size() + || (keys.size() * Math.log(children.size()) / LOG2) + > (keys.size() + children.size()) + ? mergeSearchInChildren(keys, exactSearch) + : binarySearchInChildren(keys, exactSearch); + List> pairs = new ArrayList<>(); + int previousIndex = -1; + Pair previousPair = null; + for (int idx : indexArr) { + if (previousIndex == idx) { + pairs.add(previousPair); + } else { + Pair current = getChildIndexEntry(idx); + pairs.add(current); + previousIndex = idx; + previousPair = current; + } + } + return pairs; + } + + int[] binarySearchInChildren(List keys, boolean exactSearch) { + int[] results = new int[keys.size()]; + Arrays.fill(results, -1); + int currentLow = 0; + int high = children.size() - 1; + + for (int i = 0; i < keys.size(); i++) { + Comparable key = keys.get(i); + if (currentLow > high) { + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : currentLow - 1); + return results; + } + + int foundIndex = -1; + int start = currentLow; + int end = high; + + while (start <= end) { + int mid = (start + end) >>> 1; + IMetadataIndexEntry midVal = children.get(mid); + int cmp = midVal.getCompareKey().compareTo(key); + + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + } else { + foundIndex = mid; + break; + } + } + + if (foundIndex >= 0) { + results[i] = foundIndex; + currentLow = foundIndex + 1; + } else { + if (exactSearch) { + results[i] = -1; + } else { + int insertPos = start - 1; + results[i] = insertPos; + currentLow = start; + } + } + } + return results; + } + + int[] mergeSearchInChildren(List keys, boolean exactSearch) { + int[] results = new int[keys.size()]; + int i = 0; + int j = 0; + while (i < keys.size() && j < children.size()) { + Comparable currentKey = keys.get(i); + Comparable currentChild = children.get(j).getCompareKey(); + int cmp = currentKey.compareTo(currentChild); + if (cmp == 0) { + results[i] = j; + i++; + j++; + } else if (cmp > 0) { + j++; + } else { + if (exactSearch) { + results[i] = -1; + } else { + results[i] = j - 1; + } + i++; + } + } + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : children.size() - 1); + return results; + } + public boolean isDeviceLevel() { return this.nodeType == MetadataIndexNodeType.INTERNAL_DEVICE || this.nodeType == MetadataIndexNodeType.LEAF_DEVICE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 033e3d044..cf60f7097 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -67,6 +67,8 @@ import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.controller.CachedChunkLoaderImpl; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; +import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; +import org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext; import org.apache.tsfile.read.reader.TsFileInput; import org.apache.tsfile.read.reader.page.PageReader; import org.apache.tsfile.read.reader.page.TimePageReader; @@ -535,6 +537,123 @@ private Map readDeviceMetadataFromDisk(IDeviceID dev return deviceMetadata; } + /** + * Find the offset of MetadataIndexNode corresponding to every device to avoid repeated reading of + * internal MetadataIndexNode + * + * @param table table name. ("" or null) for cases with different tables + * @param sortedDevices devices should be sorted + * @param ioSizeRecorder can be null + * @return Each element of the outer array corresponds to the device at this index. The inner + * array size is 2, the first element is the start offset, and the second is the end offset + * @throws IOException io error + */ + public DeviceMetadataIndexEntriesQueryResult getDeviceMetadataIndexNodeOffsets( + String table, List sortedDevices, LongConsumer ioSizeRecorder) throws IOException { + DeviceMetadataIndexNodeOffsetsQueryContext context = + new DeviceMetadataIndexNodeOffsetsQueryContext(sortedDevices.size(), getAllMetadataSize()); + if (sortedDevices.isEmpty()) { + return context.compact(); + } + readFileMetadata(); + // for many table + if (table == null || table.isEmpty()) { + int tableStartIndex = 0; + String previousTable = null; + for (int i = 0; i < sortedDevices.size(); i++) { + String currentTable = sortedDevices.get(i).getTableName(); + if (i == 0) { + previousTable = currentTable; + continue; + } + if (!previousTable.equals(currentTable)) { + getDeviceMetadataIndexNodeOffsets( + context, sortedDevices, tableStartIndex, i, previousTable, ioSizeRecorder); + tableStartIndex = i; + previousTable = currentTable; + } + } + // for last table + getDeviceMetadataIndexNodeOffsets( + context, + sortedDevices, + tableStartIndex, + sortedDevices.size(), + previousTable, + ioSizeRecorder); + } else { + getDeviceMetadataIndexNodeOffsets( + context, sortedDevices, 0, sortedDevices.size(), table, ioSizeRecorder); + } + return context.compact(); + } + + private void getDeviceMetadataIndexNodeOffsets( + DeviceMetadataIndexNodeOffsetsQueryContext context, + List devices, + int deviceStartIdx, + int deviceEndIdx, + String table, + LongConsumer ioSizeRecorder) + throws IOException { + MetadataIndexNode tableMetadataIndexNode = getTableRootNode(table); + if (tableMetadataIndexNode == null) { + return; + } + getDeviceMetadataIndexNodeOffsets( + context, devices, deviceStartIdx, deviceEndIdx, tableMetadataIndexNode, ioSizeRecorder); + } + + private void getDeviceMetadataIndexNodeOffsets( + DeviceMetadataIndexNodeOffsetsQueryContext context, + List devices, + int deviceStartIdx, + int deviceEndIdx, + MetadataIndexNode startNode, + LongConsumer ioSizeRecorder) + throws IOException { + MetadataIndexNodeType metadataIndexNodeType = startNode.getNodeType(); + boolean exactSearch = metadataIndexNodeType == MetadataIndexNodeType.LEAF_DEVICE; + List> entries = + startNode.getChildIndexEntries(devices.subList(deviceStartIdx, deviceEndIdx), exactSearch); + Iterator> metadataIndexEntriesIterator = entries.iterator(); + int startIdxOfChild = deviceStartIdx; + Pair previousPair = null; + for (int i = deviceStartIdx; i < deviceEndIdx; i++) { + Pair pair = metadataIndexEntriesIterator.next(); + if (exactSearch) { + if (pair != null) { + context.addDeviceMetadataIndexNodeOffset(i, pair.getLeft().getOffset(), pair.getRight()); + } + continue; + } + if (previousPair == null) { + previousPair = pair; + continue; + } + if (previousPair == pair) { + continue; + } + IMetadataIndexEntry entry = previousPair.getLeft(); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight(), ioSizeRecorder); + MetadataIndexNode lastNode = + MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); + getDeviceMetadataIndexNodeOffsets( + context, devices, startIdxOfChild, i, lastNode, ioSizeRecorder); + previousPair = pair; + startIdxOfChild = i; + } + if (exactSearch || previousPair == null) { + return; + } + // for last entry + IMetadataIndexEntry entry = previousPair.getLeft(); + ByteBuffer buffer = readData(entry.getOffset(), previousPair.getRight(), ioSizeRecorder); + MetadataIndexNode lastNode = MetadataIndexNode.deserializeFrom(buffer, true, deserializeConfig); + getDeviceMetadataIndexNodeOffsets( + context, devices, startIdxOfChild, deviceEndIdx, lastNode, ioSizeRecorder); + } + public TimeseriesMetadata readTimeseriesMetadata( IDeviceID device, String measurement, boolean ignoreNotExists) throws IOException { return readTimeseriesMetadata(device, measurement, ignoreNotExists, null); @@ -546,11 +665,31 @@ public TimeseriesMetadata readTimeseriesMetadata( boolean ignoreNotExistDevice, LongConsumer ioSizeConsumer) throws IOException { + return readTimeseriesMetadata(device, null, measurement, ignoreNotExistDevice, ioSizeConsumer); + } + + public TimeseriesMetadata readTimeseriesMetadata( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + String measurement, + boolean ignoreNotExistDevice, + LongConsumer ioSizeConsumer) + throws IOException { readFileMetadata(ioSizeConsumer); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeConsumer); + Pair metadataIndexPair; + if (deviceMetadataIndexNodeOffset == null) { + metadataIndexPair = + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeConsumer); + } else { + metadataIndexPair = + new Pair<>( + new DeviceMetadataIndexEntry(device, deviceMetadataIndexNodeOffset[0]), + deviceMetadataIndexNodeOffset[1]); + } + if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return null; @@ -683,13 +822,15 @@ public ITimeSeriesMetadata readITimeseriesMetadata(Path path, boolean ignoreNotE */ public List readTimeseriesMetadata( IDeviceID device, + long[] deviceMetadataIndexNodeOffset, String measurement, Set allSensors, boolean ignoreNotExistDevice, LongConsumer ioSizeRecorder) throws IOException { Pair metadataIndexPair = - getLeafMetadataIndexPair(device, measurement, ioSizeRecorder); + getLeafMetadataIndexPair( + device, deviceMetadataIndexNodeOffset, measurement, ioSizeRecorder); if (metadataIndexPair == null) { if (ignoreNotExistDevice) { return Collections.emptyList(); @@ -744,17 +885,28 @@ public List readTimeseriesMetadata( /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( - IDeviceID device, String measurement, LongConsumer ioSizeRecorder) throws IOException { + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + String measurement, + LongConsumer ioSizeRecorder) + throws IOException { readFileMetadata(ioSizeRecorder); MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); - Pair metadataIndexPair = - getMetadataAndEndOffsetOfDeviceNode(deviceMetadataIndexNode, device, true, ioSizeRecorder); - if (metadataIndexPair == null) { - return null; + Pair metadataIndexPair = null; + ByteBuffer buffer; + if (deviceMetadataIndexNodeOffset == null) { + metadataIndexPair = + getMetadataAndEndOffsetOfDeviceNode( + deviceMetadataIndexNode, device, true, ioSizeRecorder); + if (metadataIndexPair == null) { + return null; + } + buffer = + readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); + } else { + buffer = readData(deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1]); } - ByteBuffer buffer = - readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right, ioSizeRecorder); MetadataIndexNode metadataIndexNode = deviceMetadataIndexNode; if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { try { diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java new file mode 100644 index 000000000..0786c504a --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexEntriesQueryResult.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.query; + +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; + +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_BYTE; +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_INTEGER; +import static org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext.MAX_UNSIGNED_SHORT; + +public interface DeviceMetadataIndexEntriesQueryResult extends Accountable { + long[] getDeviceMetadataIndexNodeOffset(int deviceIndex); + + int length(); +} + +class ArrDeviceMetadataIndexEntriesQueryResult implements DeviceMetadataIndexEntriesQueryResult { + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ArrDeviceMetadataIndexEntriesQueryResult.class); + + private final Object offsetDeltaArr; + private final Object nodeSizeArr; + private final long standardOffset; + private final int length; + + ArrDeviceMetadataIndexEntriesQueryResult( + Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { + this.offsetDeltaArr = offsetDeltaArr; + this.nodeSizeArr = nodeSizeArr; + this.standardOffset = standardOffset; + this.length = length; + } + + @Override + public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { + if (deviceIndex >= length) { + return null; + } + long nodeSize; + long startOffset; + if (nodeSizeArr instanceof byte[]) { + nodeSize = (((byte[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_BYTE; + } else if (nodeSizeArr instanceof short[]) { + nodeSize = (((short[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_SHORT; + } else if (nodeSizeArr instanceof int[]) { + nodeSize = (((int[]) nodeSizeArr)[deviceIndex]) & MAX_UNSIGNED_INTEGER; + } else { + nodeSize = ((long[]) nodeSizeArr)[deviceIndex]; + } + if (nodeSize <= 0) { + return null; + } + if (offsetDeltaArr instanceof short[]) { + startOffset = standardOffset + (((short[]) offsetDeltaArr)[deviceIndex] & MAX_UNSIGNED_SHORT); + } else if (offsetDeltaArr instanceof int[]) { + startOffset = standardOffset + (((int[]) offsetDeltaArr)[deviceIndex] & MAX_UNSIGNED_INTEGER); + } else { + startOffset = standardOffset + ((long[]) offsetDeltaArr)[deviceIndex]; + } + + return new long[] {startOffset, startOffset + nodeSize}; + } + + @Override + public int length() { + return length; + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + getRamBytesUsedOfArr(nodeSizeArr) + getRamBytesUsedOfArr(offsetDeltaArr); + } + + private long getRamBytesUsedOfArr(Object arr) { + if (arr instanceof long[]) { + return RamUsageEstimator.sizeOfLongArray(length); + } else if (arr instanceof int[]) { + return RamUsageEstimator.sizeOfIntArray(length); + } else if (arr instanceof short[]) { + return RamUsageEstimator.sizeOfShortArray(length); + } else { + return RamUsageEstimator.sizeOfByteArray(length); + } + } +} + +class MapDeviceMetadataIndexEntriesQueryResult implements DeviceMetadataIndexEntriesQueryResult { + + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(MapDeviceMetadataIndexEntriesQueryResult.class); + private final Object indexMap; + private final ArrDeviceMetadataIndexEntriesQueryResult arrResult; + + public MapDeviceMetadataIndexEntriesQueryResult( + Object indexMap, Object offsetDeltaArr, Object nodeSizeArr, long standardOffset, int length) { + this.indexMap = indexMap; + this.arrResult = + new ArrDeviceMetadataIndexEntriesQueryResult( + offsetDeltaArr, nodeSizeArr, standardOffset, length); + } + + @Override + public long[] getDeviceMetadataIndexNodeOffset(int deviceIndex) { + if (arrResult.length() == 0) { + return null; + } + int idx; + if (indexMap instanceof int[]) { + idx = + Arrays.binarySearch( + (int[]) indexMap, 0, Math.min(deviceIndex + 1, arrResult.length()), deviceIndex); + } else { + idx = + unsignedBinarySearch( + (short[]) indexMap, 0, Math.min(deviceIndex + 1, arrResult.length()), deviceIndex); + } + if (idx < 0) { + return null; + } + return arrResult.getDeviceMetadataIndexNodeOffset(idx); + } + + private int unsignedBinarySearch(short[] arr, int fromIndex, int toIndex, int key) { + int low = fromIndex; + int high = toIndex - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int midVal = arr[mid] & (int) MAX_UNSIGNED_SHORT; + + if (midVal < key) { + low = mid + 1; + } else if (midVal > key) { + high = mid - 1; + } else { + return mid; + } // key found + } + return -(low + 1); // key not found. + } + + @Override + public int length() { + return arrResult.length(); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + + arrResult.ramBytesUsed() + + (indexMap instanceof int[] + ? RamUsageEstimator.sizeOfIntArray(arrResult.length()) + : RamUsageEstimator.sizeOfShortArray(arrResult.length())); + } +} diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java new file mode 100644 index 000000000..43cf4cee5 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/DeviceMetadataIndexNodeOffsetsQueryContext.java @@ -0,0 +1,389 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.query; + +import org.apache.tsfile.utils.RamUsageEstimator; + +public class DeviceMetadataIndexNodeOffsetsQueryContext { + public static final long MAX_UNSIGNED_INTEGER = 0XFFFFFFFFL; + public static final long MAX_UNSIGNED_SHORT = 0XFFFFL; + public static final long MAX_UNSIGNED_BYTE = 0XFFL; + + private long standardStartOffset; + private long minStartOffset = Long.MAX_VALUE; + private long maxStartOffset = Long.MIN_VALUE; + private long maxStartOffsetDelta; + private long maxNodeSize; + private final int length; + private int used; + + private long[] longStartOffsetArr; + private long[] longNodeSizeArr; + private int[] intNodeSizeArr; + private short[] shortNodeSizeArr; + private byte[] byteNodeSizeArr; + + private Object startOffsetDeltaArr; + private Object nodeSizeArr; + + public DeviceMetadataIndexNodeOffsetsQueryContext(int length) { + this(length, -1); + } + + public DeviceMetadataIndexNodeOffsetsQueryContext(int length, long metadataSize) { + this.length = length; + this.used = 0; + longStartOffsetArr = new long[length]; + if (metadataSize <= 0 || metadataSize > MAX_UNSIGNED_INTEGER) { + longNodeSizeArr = new long[length]; + } else if (metadataSize > MAX_UNSIGNED_SHORT) { + intNodeSizeArr = new int[length]; + } else if (metadataSize > MAX_UNSIGNED_BYTE) { + shortNodeSizeArr = new short[length]; + } else { + byteNodeSizeArr = new byte[length]; + } + } + + public void addDeviceMetadataIndexNodeOffset(int i, long startOffset, long endOffset) { + minStartOffset = Math.min(startOffset, minStartOffset); + maxStartOffset = Math.max(startOffset, maxStartOffset); + longStartOffsetArr[i] = startOffset; + long nodeSize = endOffset - startOffset; + maxNodeSize = Math.max(nodeSize, maxNodeSize); + if (longNodeSizeArr != null) { + longNodeSizeArr[i] = nodeSize; + } else if (intNodeSizeArr != null) { + intNodeSizeArr[i] = (int) (nodeSize & MAX_UNSIGNED_INTEGER); + } else if (shortNodeSizeArr != null) { + shortNodeSizeArr[i] = (short) (nodeSize & MAX_UNSIGNED_SHORT); + } else { + byteNodeSizeArr[i] = (byte) (nodeSize & MAX_UNSIGNED_BYTE); + } + used++; + } + + public DeviceMetadataIndexEntriesQueryResult compact() { + maxStartOffsetDelta = maxStartOffset - minStartOffset; + boolean compactToMap = estimateMapRamBytes() < estimateArrRamBytes(length); + return compactToMap ? compactToIntMap() : compactToArr(); + } + + private long estimateMapRamBytes() { + return estimateArrRamBytes(used) + + (length <= MAX_UNSIGNED_SHORT + ? RamUsageEstimator.sizeOfShortArray(used) + : RamUsageEstimator.sizeOfIntArray(used)); + } + + private long estimateArrRamBytes(int arrLength) { + long cost = 0; + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { + cost += RamUsageEstimator.sizeOfShortArray(arrLength); + } else if (maxStartOffsetDelta <= MAX_UNSIGNED_INTEGER) { + cost += RamUsageEstimator.sizeOfIntArray(arrLength); + } else { + cost += RamUsageEstimator.sizeOfLongArray(arrLength); + } + if (maxNodeSize <= MAX_UNSIGNED_BYTE) { + cost += RamUsageEstimator.sizeOfByteArray(arrLength); + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT) { + cost += RamUsageEstimator.sizeOfShortArray(arrLength); + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER) { + cost += RamUsageEstimator.sizeOfIntArray(arrLength); + } else { + cost += RamUsageEstimator.sizeOfLongArray(arrLength); + } + return cost; + } + + private DeviceMetadataIndexEntriesQueryResult compactToArr() { + standardStartOffset = minStartOffset; + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { + short[] shortStartOffsetDeltaArr = new short[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + shortStartOffsetDeltaArr[i] = + (short) (MAX_UNSIGNED_SHORT & (longStartOffsetArr[i] - standardStartOffset)); + } + startOffsetDeltaArr = shortStartOffsetDeltaArr; + } else if (maxStartOffsetDelta < Integer.MAX_VALUE) { + int[] intStartOffsetDeltaArr = new int[length]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + intStartOffsetDeltaArr[i] = + (int) (MAX_UNSIGNED_INTEGER & (longStartOffsetArr[i] - standardStartOffset)); + } + startOffsetDeltaArr = intStartOffsetDeltaArr; + } else { + standardStartOffset = 0; + startOffsetDeltaArr = longStartOffsetArr; + } + longStartOffsetArr = null; + if (maxNodeSize <= MAX_UNSIGNED_BYTE && byteNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToByteArr(); + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT && shortNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToShortArr(); + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER && intNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToIntArr(); + } else { + nodeSizeArr = longNodeSizeArr; + } + clearDeprecatedNodeSizeArr(); + + return new ArrDeviceMetadataIndexEntriesQueryResult( + startOffsetDeltaArr, nodeSizeArr, standardStartOffset, length); + } + + private DeviceMetadataIndexEntriesQueryResult compactToIntMap() { + int[] intIndexMap = null; + short[] shortIndexMap = null; + if (length <= Short.MAX_VALUE) { + shortIndexMap = new short[used]; + } else { + intIndexMap = new int[used]; + } + int mapSize = 0; + standardStartOffset = minStartOffset; + if (maxStartOffsetDelta <= MAX_UNSIGNED_SHORT) { + short[] shortStartOffsetDeltaArr = new short[used]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + shortStartOffsetDeltaArr[mapSize] = + (short) (MAX_UNSIGNED_SHORT & (longStartOffsetArr[i] - standardStartOffset)); + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); + } + } + startOffsetDeltaArr = shortStartOffsetDeltaArr; + } else if (maxStartOffsetDelta <= MAX_UNSIGNED_INTEGER) { + int[] intStartOffsetDeltaArr = new int[used]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + intStartOffsetDeltaArr[mapSize] = + (int) (MAX_UNSIGNED_INTEGER & (longStartOffsetArr[i] - standardStartOffset)); + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); + } + } + startOffsetDeltaArr = intStartOffsetDeltaArr; + } else { + long[] newLongStartOffsetDeltaArr = new long[used]; + for (int i = 0; i < length; i++) { + if (longStartOffsetArr[i] <= 0) { + continue; + } + newLongStartOffsetDeltaArr[mapSize] = longStartOffsetArr[i]; + if (intIndexMap != null) { + intIndexMap[mapSize++] = i; + } else { + shortIndexMap[mapSize++] = (short) (i & MAX_UNSIGNED_SHORT); + } + } + startOffsetDeltaArr = newLongStartOffsetDeltaArr; + standardStartOffset = 0; + } + longStartOffsetArr = null; + + if (maxNodeSize <= MAX_UNSIGNED_BYTE && byteNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToByteArr(used); + } else if (maxNodeSize <= MAX_UNSIGNED_SHORT && shortNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToShortArr(used); + } else if (maxNodeSize <= MAX_UNSIGNED_INTEGER && intNodeSizeArr == null) { + nodeSizeArr = compactNodeSizeArrToIntArr(used); + } else { + nodeSizeArr = compactNodeSizeToLongArr(used); + } + clearDeprecatedNodeSizeArr(); + + return new MapDeviceMetadataIndexEntriesQueryResult( + intIndexMap == null ? shortIndexMap : intIndexMap, + startOffsetDeltaArr, + nodeSizeArr, + standardStartOffset, + used); + } + + private void clearDeprecatedNodeSizeArr() { + longNodeSizeArr = null; + intNodeSizeArr = null; + shortNodeSizeArr = null; + byteNodeSizeArr = null; + } + + private int[] compactNodeSizeArrToIntArr() { + if (intNodeSizeArr != null) { + return intNodeSizeArr; + } + int[] newIntNodeSizeArr = new int[length]; + for (int i = 0; i < length; i++) { + newIntNodeSizeArr[i] = (int) (longNodeSizeArr[i] & MAX_UNSIGNED_INTEGER); + } + return newIntNodeSizeArr; + } + + private short[] compactNodeSizeArrToShortArr() { + shortNodeSizeArr = shortNodeSizeArr == null ? new short[length] : shortNodeSizeArr; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + shortNodeSizeArr[i] = (short) (longNodeSizeArr[i] & MAX_UNSIGNED_SHORT); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + shortNodeSizeArr[i] = (short) (intNodeSizeArr[i] & MAX_UNSIGNED_SHORT); + } + } + return shortNodeSizeArr; + } + + private byte[] compactNodeSizeArrToByteArr() { + byteNodeSizeArr = byteNodeSizeArr == null ? new byte[length] : byteNodeSizeArr; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (longNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (intNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + intNodeSizeArr = null; + } else if (shortNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + byteNodeSizeArr[i] = (byte) (shortNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + shortNodeSizeArr = null; + } + return byteNodeSizeArr; + } + + private long[] compactNodeSizeToLongArr(int newLength) { + long[] newLongNodeSizeArr = new long[newLength]; + int j = 0; + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newLongNodeSizeArr[j++] = longNodeSizeArr[i]; + } + return newLongNodeSizeArr; + } + + private int[] compactNodeSizeArrToIntArr(int newLength) { + int[] intNodeSizeArr = new int[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + intNodeSizeArr[j++] = (int) (longNodeSizeArr[i] & MAX_UNSIGNED_INTEGER); + } + } else { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + intNodeSizeArr[j++] = intNodeSizeArr[i]; + } + } + return intNodeSizeArr; + } + + private short[] compactNodeSizeArrToShortArr(int newLength) { + short[] newShortNodeSizeArr = new short[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = (short) (longNodeSizeArr[i] & MAX_UNSIGNED_SHORT); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = (short) (intNodeSizeArr[i] & MAX_UNSIGNED_SHORT); + } + } else { + for (int i = 0; i < length; i++) { + if (shortNodeSizeArr[i] == 0) { + continue; + } + newShortNodeSizeArr[j++] = shortNodeSizeArr[i]; + } + } + return newShortNodeSizeArr; + } + + private byte[] compactNodeSizeArrToByteArr(int newLength) { + byte[] newByteNodeSizeArr = new byte[newLength]; + int j = 0; + if (longNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (longNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (longNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + longNodeSizeArr = null; + } else if (intNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (intNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (intNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + intNodeSizeArr = null; + } else if (shortNodeSizeArr != null) { + for (int i = 0; i < length; i++) { + if (shortNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = (byte) (shortNodeSizeArr[i] & MAX_UNSIGNED_BYTE); + } + shortNodeSizeArr = null; + } else { + for (int i = 0; i < length; i++) { + if (byteNodeSizeArr[i] == 0) { + continue; + } + newByteNodeSizeArr[j++] = byteNodeSizeArr[i]; + } + } + return newByteNodeSizeArr; + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java new file mode 100644 index 000000000..51e019f97 --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/DeviceMetadataIndexEntriesQueryTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MetadataIndexNode; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.query.DeviceMetadataIndexEntriesQueryResult; +import org.apache.tsfile.read.query.DeviceMetadataIndexNodeOffsetsQueryContext; +import org.apache.tsfile.utils.FileGenerator; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class DeviceMetadataIndexEntriesQueryTest { + private static final String FILE_PATH = FileGenerator.outputDataFile; + + @Before + public void before() throws IOException {} + + @After + public void after() throws IOException { + Files.deleteIfExists(new File(FILE_PATH).toPath()); + } + + @Test + public void test1() throws IOException, WriteProcessException { + File file = new File(FILE_PATH); + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("device", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT32)), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); + try (ITsFileWriter writer = + new TsFileWriterBuilder().tableSchema(tableSchema).file(file).build()) { + Tablet tablet = + new Tablet( + Arrays.asList("device", "s1"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32), + 10000); + for (int i = 0; i < 10000; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("device", i, "d" + i); + tablet.addValue("s1", i, i); + } + writer.write(tablet); + } + + List queriedDevices = new ArrayList<>(); + for (int i = 0; i < 20000; i++) { + if (i >= 15000) { + queriedDevices.add(new StringArrayDeviceID("t2.d" + i)); + } else { + queriedDevices.add(new StringArrayDeviceID("t1.d" + i)); + } + } + queriedDevices.sort(IDeviceID::compareTo); + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + DeviceMetadataIndexEntriesQueryResult offsets = + reader.getDeviceMetadataIndexNodeOffsets(null, queriedDevices, null); + for (int i = 0; i < queriedDevices.size(); i++) { + IDeviceID deviceID = queriedDevices.get(i); + int deviceNumber = Integer.parseInt(deviceID.toString().substring("t1.d".length())); + long[] metadataIndexNodeOffsetOfCurDevice = offsets.getDeviceMetadataIndexNodeOffset(i); + if (deviceNumber >= 10000) { + Assert.assertNull(metadataIndexNodeOffsetOfCurDevice); + continue; + } + MetadataIndexNode metadataIndexNode = + reader.readMetadataIndexNode( + metadataIndexNodeOffsetOfCurDevice[0], + metadataIndexNodeOffsetOfCurDevice[1], + false); + List alignedChunkMetadataList = + reader.getAlignedChunkMetadataByMetadataIndexNode(deviceID, metadataIndexNode, true); + Assert.assertEquals(1, alignedChunkMetadataList.size()); + + Assert.assertEquals(deviceNumber, alignedChunkMetadataList.get(0).getStartTime()); + } + + Assert.assertEquals( + 0, + reader.getDeviceMetadataIndexNodeOffsets("t1", Collections.emptyList(), null).length()); + offsets = + reader.getDeviceMetadataIndexNodeOffsets( + "t1", Collections.singletonList(new StringArrayDeviceID("t1.d")), null); + Assert.assertNull(offsets.getDeviceMetadataIndexNodeOffset(0)); + } + } + + @Test + public void test2() { + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(1); + queryContext.addDeviceMetadataIndexNodeOffset(0, 1, (long) Integer.MAX_VALUE + 10); + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + long[] offsets = result1.getDeviceMetadataIndexNodeOffset(0); + Assert.assertEquals(1, offsets[0]); + Assert.assertEquals((long) Integer.MAX_VALUE + 10, offsets[1]); + } + + @Test + public void test3() { + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(10); + for (int i = 0; i < 10; i++) { + queryContext.addDeviceMetadataIndexNodeOffset(i, i + 1, 0XFFFFFFFFL + 10 + i); + } + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + for (int i = 0; i < 10; i++) { + Assert.assertEquals(i + 1, result1.getDeviceMetadataIndexNodeOffset(i)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + i, result1.getDeviceMetadataIndexNodeOffset(i)[1]); + } + queryContext = new DeviceMetadataIndexNodeOffsetsQueryContext(10); + queryContext.addDeviceMetadataIndexNodeOffset(0, 1, 0XFFFFFFFFL + 10 + 10); + DeviceMetadataIndexEntriesQueryResult result2 = queryContext.compact(); + Assert.assertTrue(result1.ramBytesUsed() > result2.ramBytesUsed()); + Assert.assertEquals(1, result2.getDeviceMetadataIndexNodeOffset(0)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + 10, result2.getDeviceMetadataIndexNodeOffset(0)[1]); + } + + @Test + public void test4() { + int length = (int) Short.MAX_VALUE * 2; + DeviceMetadataIndexNodeOffsetsQueryContext queryContext = + new DeviceMetadataIndexNodeOffsetsQueryContext(length); + for (int i = 0; i < length; i++) { + if (i % 2 == 0) { + queryContext.addDeviceMetadataIndexNodeOffset(i, i + 1, 0XFFFFFFFFL + 10 + i); + } + } + DeviceMetadataIndexEntriesQueryResult result1 = queryContext.compact(); + for (int i = 0; i < length; i++) { + if (i % 2 != 0) { + Assert.assertNull(result1.getDeviceMetadataIndexNodeOffset(i)); + continue; + } + Assert.assertEquals(i + 1, result1.getDeviceMetadataIndexNodeOffset(i)[0]); + Assert.assertEquals(0XFFFFFFFFL + 10 + i, result1.getDeviceMetadataIndexNodeOffset(i)[1]); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index e4bfbb44d..750c53feb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -73,7 +73,8 @@ public void testReadTimeseriesMetadata() throws IOException { // s4 should not be returned as result set.add("s4"); List timeseriesMetadataList = - reader.readTimeseriesMetadata(path.getIDeviceID(), path.getMeasurement(), set, false, null); + reader.readTimeseriesMetadata( + path.getIDeviceID(), null, path.getMeasurement(), set, false, null); Assert.assertEquals(3, timeseriesMetadataList.size()); for (int i = 1; i <= timeseriesMetadataList.size(); i++) { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 1).getMeasurementId()); @@ -87,7 +88,8 @@ public void testReadTimeseriesMetadata() throws IOException { // so the result is not supposed to contain this measurement's timeseries metadata set.add("s8"); timeseriesMetadataList = - reader.readTimeseriesMetadata(path.getIDeviceID(), path.getMeasurement(), set, false, null); + reader.readTimeseriesMetadata( + path.getIDeviceID(), null, path.getMeasurement(), set, false, null); Assert.assertEquals(2, timeseriesMetadataList.size()); for (int i = 5; i < 7; i++) { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 5).getMeasurementId());