From 740d0971a62601ba863c63915ced8fc3aee35209 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 08:30:53 -0800 Subject: [PATCH 1/8] ignore .log files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3cf3af6..75fa5f8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *.aliases *.lvlps *.tdms_index +*.log From 94191202dea8b87c7c768e80b2601863085d0475 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 08:31:18 -0800 Subject: [PATCH 2/8] remove Windows path separators from README --- tests/defragment.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/defragment.rs diff --git a/tests/defragment.rs b/tests/defragment.rs new file mode 100644 index 0000000..97d6fc1 --- /dev/null +++ b/tests/defragment.rs @@ -0,0 +1,70 @@ +mod common; +use tedium::{ChannelPath, PropertyPath, PropertyValue, TdmsFile}; + + +#[test] +fn test_defragment() { + // get the test file + let mut input_file = common::open_test_file(); + + // create a new tdms file for our defragmented output + let path = std::path::Path::new("tests/test_defragment_output.tdms"); + let mut output_file = TdmsFile::create(path).unwrap(); + + // get the properties of the input file and write them to the properties of the output file + let mut all_properties = input_file.read_all_properties(&PropertyPath::file()).unwrap(); + + let mut writer = output_file.writer().unwrap(); + + // convert all_properties to a `&[(&str, PropertyValue)]` we can pass to the writer.write_properties method + let mut properties: Vec<(&str, PropertyValue)> = Vec::new(); + for (key, value) in all_properties.iter() { + let property = value.clone(); + properties.push((key.as_str(), property.clone())); + } + + writer.write_properties(&PropertyPath::file(), &properties).unwrap(); + + // First, collect all group names + let groups: Vec = input_file.list_groups().into_iter().collect(); + + // print the number of groups + println!("num groups: {:?}", groups.len()); + + // Then iterate over the collected group names + for group_path in groups { + + // print the group path + println!("group_path: {:?}", group_path); + + // Collect channel names for each group before mutating `input_file` + let channels: Vec = input_file.list_channels_in_group(&group_path).into_iter().collect(); + + // print the number of channels + println!("num channels: {:?}", channels.len()); + + for channel_path in channels { + + // print the channel path + println!("channel_path: {:?}", channel_path); + + let channel_length = input_file.channel_length(&channel_path).unwrap(); + + // print the channel length + println!("channel_length: {:?}", channel_length); + + // create a buffer to hold the channel data, and read the channel data into it + let mut data = vec![0.0; channel_length as usize]; + + // Since we're no longer borrowing `input_file` to list groups/channels, this mutable borrow is okay + input_file.read_channel(&channel_path, &mut data).unwrap(); + + // print the data + println!("data: {:?}", data); + + // Write the channel data to the output file + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + } + } + +} From 816c40d33eb6e27d064bf8c41ad961ebb8048099 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 08:31:38 -0800 Subject: [PATCH 3/8] typo in comment --- tests/write_properties.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/write_properties.rs b/tests/write_properties.rs index 0dbd38a..8e6b5b2 100644 --- a/tests/write_properties.rs +++ b/tests/write_properties.rs @@ -35,7 +35,7 @@ fn write_properties(file: &mut TdmsFile, path let mut writer = file.writer().unwrap(); writer.write_properties(path, TEST_PROPERTIES).unwrap(); - //this one wont exist as a constant. + //this one won't exist as a constant. writer .write_properties( &path, From d64355d007502c904e660f65f56896dbb5588f3d Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 08:32:27 -0800 Subject: [PATCH 4/8] removed windows path separator --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3c39eba..63f6c6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://github.com/WiresmithTech/tedium" repository = "https://github.com/WiresmithTech/tedium" description = "A fast and easy TDMS Library for Rust" keywords = ["labview", "ni", "tdms"] -readme = ".\\README.md" +readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 81449cf3d86ff4b279dac296e82f664fcce43fc0 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 08:35:39 -0800 Subject: [PATCH 5/8] test file for defragment output --- tests/test_defragment.tdms | Bin 0 -> 3064 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_defragment.tdms diff --git a/tests/test_defragment.tdms b/tests/test_defragment.tdms new file mode 100644 index 0000000000000000000000000000000000000000..c655d684d7db9266fce5a141ad7506f619615e14 GIT binary patch literal 3064 zcmeHJJ5Iwu5S>86Cwv5mf(nHLL@2hkN*N*7G^C=5OuUvXc@qn-rCb4VDT|70Kuw*7 zBD}Z31qYzpCyi&`%xY(6WzEhselR;Cl3$3-J1~G&lTvOsz&VUQ;dgDkKP<`R}j;g2&`pI$tSN50%e$%~3j2=LrLTqZP zq1BZgW_h8R(&h#M`{nMza?R?4|3VYRK&WjAp{fG;~u`fslR@I g{r>Ow&Br$%-}~_(_@~!Dy?*lHOh)m8=f4e}KdHwv{{R30 literal 0 HcmV?d00001 From 8bfd04b8dc49b59d2e40de34fa92f8194536e1b8 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 09:23:17 -0800 Subject: [PATCH 6/8] added copy_channel method --- tests/defragment.rs | 66 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/tests/defragment.rs b/tests/defragment.rs index 97d6fc1..363831b 100644 --- a/tests/defragment.rs +++ b/tests/defragment.rs @@ -1,5 +1,62 @@ mod common; -use tedium::{ChannelPath, PropertyPath, PropertyValue, TdmsFile}; +use tedium::{ChannelPath, PropertyPath, PropertyValue, TdmsFile, TdmsStorageType}; + + +// Comment from the tedium author: +// +// So it should just adapt to the type of your array. There is a trait TdmsStorageType which is implemented for supported types for channels. So if you just use an array of f64 for example it will adapt to that. +// +// Now I can imagine how that could go wrong with defrag, if read is generic, and write is generic - it has to resolve to a type somewhere! +// You almost want a copy_channel method which is generic and then above that check the type and call the right version. +// +// fn copy_channel_data(source, dest, channel) { +// generic_read(), +// generic_write() +// } +// +// fn copy_channel(source, dest, channel) { +// let type = source.get_channel_type(); +// match type { +// f64 => copy_channel_data::(source, dest, channel) +// } +// } + +// let's implement the copy_channel_data method +fn copy_channel_data(source: &mut TdmsFile, dest: &mut TdmsFile, channel: &ChannelPath) { + let channel_length = source.channel_length(channel).unwrap(); + let mut data = vec![0.0; channel_length as usize]; + source.read_channel(channel, &mut data).unwrap(); + dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); +} + +// let's implement the copy_channel method +fn copy_channel(source: &mut TdmsFile, dest: &mut TdmsFile, channel: &ChannelPath) { + let channel_type = source.channel_type(channel).unwrap(); + match channel_type { + TdmsStorageType::Double => copy_channel_data::(source, dest, channel), + TdmsStorageType::Single => copy_channel_data::(source, dest, channel), + TdmsStorageType::Int64 => copy_channel_data::(source, dest, channel), + TdmsStorageType::Int32 => copy_channel_data::(source, dest, channel), + TdmsStorageType::Int16 => copy_channel_data::(source, dest, channel), + TdmsStorageType::Int8 => copy_channel_data::(source, dest, channel), + TdmsStorageType::UInt64 => copy_channel_data::(source, dest, channel), + TdmsStorageType::UInt32 => copy_channel_data::(source, dest, channel), + TdmsStorageType::UInt16 => copy_channel_data::(source, dest, channel), + TdmsStorageType::UInt8 => copy_channel_data::(source, dest, channel), + TdmsStorageType::String => { + let channel_length = source.channel_length(channel).unwrap(); + let mut data = vec![String::new(); channel_length as usize]; + source.read_channel(channel, &mut data).unwrap(); + dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); + } + TdmsStorageType::Boolean => { + let channel_length = source.channel_length(channel).unwrap(); + let mut data = vec![false; channel_length as usize]; + source.read_channel(channel, &mut data).unwrap(); + dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); + } + } +} #[test] @@ -62,8 +119,11 @@ fn test_defragment() { // print the data println!("data: {:?}", data); - // Write the channel data to the output file - writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + // // Write the channel data to the output file + // writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + + // copy the channel data to the output file + copy_channel(&mut input_file, &mut output_file, &channel_path); } } From 6724aefb79c857e3d4fed4a92ac1e1a8617e529c Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 10:30:00 -0800 Subject: [PATCH 7/8] support for getting channel DataType --- src/file/mod.rs | 25 ++++++++++++++++++++++++- src/index/mod.rs | 19 ++++++++++++++++++- src/lib.rs | 2 +- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/file/mod.rs b/src/file/mod.rs index 39fd73e..35e25b7 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use crate::meta_data::Segment; -use crate::{error::TdmsError, PropertyPath, PropertyValue}; +use crate::{DataType, error::TdmsError, PropertyPath, PropertyValue}; use crate::{index::Index, ChannelPath}; use crate::{ io::writer::{LittleEndianWriter, TdmsWriter}, @@ -149,6 +149,11 @@ impl TdmsFile { let paths = self.index.paths_starting_with(group.path()); paths.filter_map(|path| ChannelPath::try_from(path).ok()) } + + pub fn get_channel_type(&self, channel: &ChannelPath) -> Option { + let data_type = *self.index.channel_type(channel).unwrap(); + Some(data_type) + } } impl TdmsFile { @@ -339,4 +344,22 @@ mod tests { .collect(); assert_eq!(channels.len(), 0); } + + #[test] + fn test_get_channel_type() { + let mut file = new_empty_file(); + + let mut writer = file.writer().unwrap(); + writer + .write_channels( + &[ChannelPath::new("group", "channel")], + &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + DataLayout::Interleaved, + ) + .unwrap(); + + drop(writer); + let channel_type = file.get_channel_type(&ChannelPath::new("group", "channel")).unwrap(); + assert_eq!(channel_type, DataType::DoubleFloat); + } } diff --git a/src/index/mod.rs b/src/index/mod.rs index 8b2744d..2938d36 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -15,7 +15,7 @@ use crate::error::TdmsError; use crate::meta_data::{ObjectMetaData, RawDataIndex, RawDataMeta}; use crate::paths::{ChannelPath, PropertyPath}; use crate::raw_data::DataBlock; -use crate::PropertyValue; +use crate::{DataType, PropertyValue}; /// A store for a given channel point to the data block with its data and the index within that. #[derive(Debug, Clone, PartialEq, Eq)] @@ -173,6 +173,23 @@ impl Index { pub fn get_data_block(&self, index: usize) -> Option<&DataBlock> { self.data_blocks.get(index) } + + // Get the data type for the given channel. + /// + /// Returns None if the channel does not exist. + pub fn channel_type(&self, path: &ChannelPath) -> Option<&DataType> { + self.objects + .get(path.path()) + .and_then(|object| { + object + .latest_data_format + .as_ref() + .and_then(|format| match format { + DataFormat::RawData(meta) => Some(&meta.data_type), + }) + }) + } + } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index 9038061..dc8bebd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ mod raw_data; pub use error::TdmsError; pub use file::TdmsFile; pub use file::TdmsFileWriter; -pub use io::data_types::TdmsStorageType; +pub use io::data_types::{TdmsStorageType, DataType}; pub use paths::{ChannelPath, PropertyPath}; pub use properties::PropertyValue; pub use raw_data::DataLayout; From fb9b6e38dda0b15892ba415800e8859983b2b6d3 Mon Sep 17 00:00:00 2001 From: Jim Kring Date: Tue, 20 Feb 2024 11:32:18 -0800 Subject: [PATCH 8/8] more progress on defragment --- src/file/mod.rs | 8 ++- tests/defragment.rs | 152 +++++++++++++++++++++++--------------------- 2 files changed, 86 insertions(+), 74 deletions(-) diff --git a/src/file/mod.rs b/src/file/mod.rs index 35e25b7..3f950db 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -151,8 +151,12 @@ impl TdmsFile { } pub fn get_channel_type(&self, channel: &ChannelPath) -> Option { - let data_type = *self.index.channel_type(channel).unwrap(); - Some(data_type) + let data_type = self.index.channel_type(channel); + if let Some(data_type) = data_type { + Some(*data_type) + } else { + None + } } } diff --git a/tests/defragment.rs b/tests/defragment.rs index 363831b..395b589 100644 --- a/tests/defragment.rs +++ b/tests/defragment.rs @@ -1,62 +1,10 @@ mod common; -use tedium::{ChannelPath, PropertyPath, PropertyValue, TdmsFile, TdmsStorageType}; - - -// Comment from the tedium author: -// -// So it should just adapt to the type of your array. There is a trait TdmsStorageType which is implemented for supported types for channels. So if you just use an array of f64 for example it will adapt to that. -// -// Now I can imagine how that could go wrong with defrag, if read is generic, and write is generic - it has to resolve to a type somewhere! -// You almost want a copy_channel method which is generic and then above that check the type and call the right version. -// -// fn copy_channel_data(source, dest, channel) { -// generic_read(), -// generic_write() -// } -// -// fn copy_channel(source, dest, channel) { -// let type = source.get_channel_type(); -// match type { -// f64 => copy_channel_data::(source, dest, channel) -// } -// } - -// let's implement the copy_channel_data method -fn copy_channel_data(source: &mut TdmsFile, dest: &mut TdmsFile, channel: &ChannelPath) { - let channel_length = source.channel_length(channel).unwrap(); - let mut data = vec![0.0; channel_length as usize]; - source.read_channel(channel, &mut data).unwrap(); - dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); -} - -// let's implement the copy_channel method -fn copy_channel(source: &mut TdmsFile, dest: &mut TdmsFile, channel: &ChannelPath) { - let channel_type = source.channel_type(channel).unwrap(); - match channel_type { - TdmsStorageType::Double => copy_channel_data::(source, dest, channel), - TdmsStorageType::Single => copy_channel_data::(source, dest, channel), - TdmsStorageType::Int64 => copy_channel_data::(source, dest, channel), - TdmsStorageType::Int32 => copy_channel_data::(source, dest, channel), - TdmsStorageType::Int16 => copy_channel_data::(source, dest, channel), - TdmsStorageType::Int8 => copy_channel_data::(source, dest, channel), - TdmsStorageType::UInt64 => copy_channel_data::(source, dest, channel), - TdmsStorageType::UInt32 => copy_channel_data::(source, dest, channel), - TdmsStorageType::UInt16 => copy_channel_data::(source, dest, channel), - TdmsStorageType::UInt8 => copy_channel_data::(source, dest, channel), - TdmsStorageType::String => { - let channel_length = source.channel_length(channel).unwrap(); - let mut data = vec![String::new(); channel_length as usize]; - source.read_channel(channel, &mut data).unwrap(); - dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); - } - TdmsStorageType::Boolean => { - let channel_length = source.channel_length(channel).unwrap(); - let mut data = vec![false; channel_length as usize]; - source.read_channel(channel, &mut data).unwrap(); - dest.write_channels(&[channel.clone()], &data, tedium::DataLayout::Interleaved).unwrap(); - } - } -} +use tedium::{ChannelPath, PropertyPath, PropertyValue, TdmsFile, DataType, TdmsFileWriter}; +use std::{ + fs::File, + io::{Read, Seek, SeekFrom, Write}, + path::Path, +}; #[test] @@ -91,6 +39,8 @@ fn test_defragment() { // Then iterate over the collected group names for group_path in groups { + println!("==== GROUP BEGIN ====="); + // print the group path println!("group_path: {:?}", group_path); @@ -102,29 +52,87 @@ fn test_defragment() { for channel_path in channels { + println!("---- CHANNEL BEGIN ----"); + // print the channel path println!("channel_path: {:?}", channel_path); + // #todo: refactor this into a `copy_channel()` function + let channel_length = input_file.channel_length(&channel_path).unwrap(); // print the channel length println!("channel_length: {:?}", channel_length); - // create a buffer to hold the channel data, and read the channel data into it - let mut data = vec![0.0; channel_length as usize]; - - // Since we're no longer borrowing `input_file` to list groups/channels, this mutable borrow is okay - input_file.read_channel(&channel_path, &mut data).unwrap(); - - // print the data - println!("data: {:?}", data); - // // Write the channel data to the output file - // writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); - - // copy the channel data to the output file - copy_channel(&mut input_file, &mut output_file, &channel_path); - } + // print the channel type + let channel_type = input_file.get_channel_type(&channel_path); + match channel_type { + Some(t) => println!("channel_type: {:?}", t), + None => println!("channel_type: None"), + } + + let channel_type = input_file.get_channel_type(&channel_path); + + match channel_type { + + Some(DataType::SingleFloat) => { + let mut data: Vec = vec![0.0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::DoubleFloat) => { + let mut data: Vec = vec![0.0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::I8) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::I16) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::I32) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::I64) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::U8) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::U16) => { + let mut data: Vec = vec![0; channel_length as usize]; + }, + Some(DataType::U32) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(DataType::U64) => { + let mut data: Vec = vec![0; channel_length as usize]; + input_file.read_channel(&channel_path, &mut data).unwrap(); + writer.write_channels(&[channel_path], &data, tedium::DataLayout::Interleaved).unwrap(); + }, + Some(data_type) => println!("Unsupported data type: {}", data_type), + None => println!("None"), + } + + println!("---- CHANNEL END ----"); + + }; + println!("==== GROUP END ====="); } + }