diff --git a/CMakeLists.txt b/CMakeLists.txt index bb5a5e92d..c8a559ad3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,8 +24,8 @@ For more information, please visit . set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") ################ PROJECT VERSION #################### -set(PROJECT_VERSION_FULL "0.4.0") -set(PROJECT_SO_VERSION 27) +set(PROJECT_VERSION_FULL "0.5.0") +set(PROJECT_SO_VERSION 28) # Remove the dash and anything following, to get the #.#.# version for project() STRING(REGEX REPLACE "\-.*$" "" VERSION_NUM "${PROJECT_VERSION_FULL}") diff --git a/examples/animation.gif b/examples/animation.gif new file mode 100644 index 000000000..0b4ca00f5 Binary files /dev/null and b/examples/animation.gif differ diff --git a/examples/domain-1d-lut.cube b/examples/domain-1d-lut.cube new file mode 100644 index 000000000..cfa2afc6d --- /dev/null +++ b/examples/domain-1d-lut.cube @@ -0,0 +1,6 @@ +TITLE "Domain 1D LUT" +DOMAIN_MIN 0.0 0.0 0.0 +DOMAIN_MAX 0.1 0.1 0.1 +LUT_1D_SIZE 2 +0.0 1.0 0.0 +0.0 0.0 1.0 diff --git a/examples/domain-3d-lut.cube b/examples/domain-3d-lut.cube new file mode 100644 index 000000000..9720e03fe --- /dev/null +++ b/examples/domain-3d-lut.cube @@ -0,0 +1,12 @@ +TITLE "Domain 3D LUT" +DOMAIN_MIN 0.0 0.0 0.0 +DOMAIN_MAX 0.1 0.1 0.1 +LUT_3D_SIZE 2 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +1.0 0.0 0.0 +0.0 0.0 1.0 diff --git a/examples/eq_sphere_plane.png b/examples/eq_sphere_plane.png new file mode 100644 index 000000000..ac574b514 Binary files /dev/null and b/examples/eq_sphere_plane.png differ diff --git a/examples/example-1d-lut.cube b/examples/example-1d-lut.cube new file mode 100644 index 000000000..dfa181126 --- /dev/null +++ b/examples/example-1d-lut.cube @@ -0,0 +1,6 @@ +TITLE "Example 1D LUT" +LUT_1D_SIZE 4 +0.000000 0.000000 0.000000 +0.300000 0.100000 0.050000 +0.600000 0.700000 0.200000 +1.000000 1.000000 1.000000 diff --git a/examples/fisheye_plane_equidistant.png b/examples/fisheye_plane_equidistant.png new file mode 100644 index 000000000..6fb2a60b7 Binary files /dev/null and b/examples/fisheye_plane_equidistant.png differ diff --git a/examples/fisheye_plane_equisolid.png b/examples/fisheye_plane_equisolid.png new file mode 100644 index 000000000..811eada21 Binary files /dev/null and b/examples/fisheye_plane_equisolid.png differ diff --git a/examples/fisheye_plane_orthographic.png b/examples/fisheye_plane_orthographic.png new file mode 100644 index 000000000..becb2e8bb Binary files /dev/null and b/examples/fisheye_plane_orthographic.png differ diff --git a/examples/fisheye_plane_stereographic.png b/examples/fisheye_plane_stereographic.png new file mode 100644 index 000000000..a72779393 Binary files /dev/null and b/examples/fisheye_plane_stereographic.png differ diff --git a/external/godot-cpp b/external/godot-cpp index 6388e26dd..cc8ad37ee 160000 --- a/external/godot-cpp +++ b/external/godot-cpp @@ -1 +1 @@ -Subproject commit 6388e26dd8a42071f65f764a3ef3d9523dda3d6e +Subproject commit cc8ad37ee0d70c70d2334f44f2eec979582061c7 diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp index 18958319b..424dc2fa7 100644 --- a/src/AudioWaveformer.cpp +++ b/src/AudioWaveformer.cpp @@ -12,13 +12,31 @@ #include "AudioWaveformer.h" +#include + +#include +#include +#include +#include +#include + +#include "Clip.h" +#include "Exceptions.h" +#include "FrameMapper.h" +#include "FFmpegReader.h" +#include "Timeline.h" + using namespace std; using namespace openshot; // Default constructor -AudioWaveformer::AudioWaveformer(ReaderBase* new_reader) : reader(new_reader) +AudioWaveformer::AudioWaveformer(ReaderBase* new_reader) : + reader(new_reader), + detached_reader(nullptr), + resolved_reader(nullptr), + source_initialized(false) { } @@ -31,104 +49,382 @@ AudioWaveformer::~AudioWaveformer() // Extract audio samples from any ReaderBase class AudioWaveformData AudioWaveformer::ExtractSamples(int channel, int num_per_second, bool normalize) { - AudioWaveformData data; - - if (reader) { - // Open reader (if needed) - bool does_reader_have_video = reader->info.has_video; - if (!reader->IsOpen()) { - reader->Open(); - } - // Disable video for faster processing - reader->info.has_video = false; - - int sample_rate = reader->info.sample_rate; - int sample_divisor = sample_rate / num_per_second; - int total_samples = num_per_second * (reader->info.duration + 1.0); - int extracted_index = 0; - - // Force output to zero elements for non-audio readers - if (!reader->info.has_audio) { - total_samples = 0; - } - - // Resize and clear audio buffers - data.resize(total_samples); - data.zero(total_samples); - - // Bail out, if no samples needed - if (total_samples == 0 || reader->info.channels == 0) { - return data; - } - - // Loop through all frames - int sample_index = 0; - float samples_max = 0.0; - float chunk_max = 0.0; - float chunk_squared_sum = 0.0; - - // How many channels are we using - int channel_count = 1; - if (channel == -1) { - channel_count = reader->info.channels; - } - - for (auto f = 1; f <= reader->info.video_length; f++) { - // Get next frame - shared_ptr frame = reader->GetFrame(f); - - // Cache channels for this frame, to reduce # of calls to frame->GetAudioSamples - float* channels[channel_count]; - for (auto channel_index = 0; channel_index < reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - channels[channel_index] = frame->GetAudioSamples(channel_index); - } - } - - // Get sample value from a specific channel (or all channels) - for (auto s = 0; s < frame->GetAudioSamplesCount(); s++) { - for (auto channel_index = 0; channel_index < reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - float *samples = channels[channel_index]; - float rms_sample_value = std::sqrt(samples[s] * samples[s]); - - // Accumulate sample averages - chunk_squared_sum += rms_sample_value; - chunk_max = std::max(chunk_max, rms_sample_value); - } - } - - sample_index += 1; - - // Cut-off reached - if (sample_index % sample_divisor == 0) { - float avg_squared_sum = chunk_squared_sum / (sample_divisor * channel_count); - data.max_samples[extracted_index] = chunk_max; - data.rms_samples[extracted_index] = avg_squared_sum; - extracted_index++; - - // Track max/min values - samples_max = std::max(samples_max, chunk_max); - - // reset sample total and index - sample_index = 0; - chunk_max = 0.0; - chunk_squared_sum = 0.0; - } - } - } - - // Scale all values to the -1 to +1 range (regardless of how small or how large the - // original audio sample values are) - if (normalize && samples_max > 0.0) { - float scale = 1.0f / samples_max; - data.scale(total_samples, scale); - } - - // Resume previous has_video value - reader->info.has_video = does_reader_have_video; - } - - - return data; + // Legacy entry point: resolve a source reader (unwrap Clip/FrameMapper), then extract audio-only. + AudioWaveformData data; + if (!reader) { + return data; + } + + ReaderBase* source = ResolveWaveformReader(); + + Fraction source_fps = ResolveSourceFPS(source); + + AudioWaveformData base = ExtractSamplesFromReader(source, channel, num_per_second, false); + + // If this is a Clip, apply its keyframes using project fps (timeline if available, else reader fps) + if (auto clip = dynamic_cast(reader)) { + Timeline* timeline = dynamic_cast(clip->ParentTimeline()); + Fraction project_fps = timeline ? timeline->info.fps : clip->Reader()->info.fps; + return ApplyKeyframes(base, &clip->time, &clip->volume, project_fps, source_fps, source->info.channels, num_per_second, channel, normalize); + } + + // No keyframes to apply + if (normalize) { + float max_sample = 0.0f; + for (auto v : base.max_samples) { + max_sample = std::max(max_sample, std::abs(v)); + } + if (max_sample > 0.0f) { + base.scale(static_cast(base.max_samples.size()), 1.0f / max_sample); + } + } + return base; +} + +AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize) { + FFmpegReader temp_reader(path); + temp_reader.Open(); + // Disable video for speed + bool has_video = temp_reader.info.has_video; + temp_reader.info.has_video = false; + AudioWaveformData data = ExtractSamplesFromReader(&temp_reader, channel, num_per_second, normalize); + temp_reader.info.has_video = has_video; + temp_reader.Close(); + return data; +} + +AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, + const Keyframe* time_keyframe, + const Keyframe* volume_keyframe, + const Fraction& project_fps, + int channel, + int num_per_second, + bool normalize) { + FFmpegReader temp_reader(path); + temp_reader.Open(); + bool has_video = temp_reader.info.has_video; + temp_reader.info.has_video = false; + Fraction source_fps = temp_reader.info.fps; + AudioWaveformData base = ExtractSamplesFromReader(&temp_reader, channel, num_per_second, false); + temp_reader.info.has_video = has_video; + temp_reader.Close(); + return ApplyKeyframes(base, time_keyframe, volume_keyframe, project_fps, source_fps, temp_reader.info.channels, num_per_second, channel, normalize); +} + +AudioWaveformData AudioWaveformer::ApplyKeyframes(const AudioWaveformData& base, + const Keyframe* time_keyframe, + const Keyframe* volume_keyframe, + const Fraction& project_fps, + const Fraction& source_fps, + int source_channels, + int num_per_second, + int channel, + bool normalize) { + AudioWaveformData data; + if (num_per_second <= 0) { + return data; + } + + double project_fps_value = project_fps.ToDouble(); + double source_fps_value = source_fps.ToDouble(); + if (project_fps_value <= 0.0 || source_fps_value <= 0.0) { + return data; + } + + if (channel != -1 && (channel < 0 || channel >= source_channels)) { + return data; + } + + size_t base_total = base.max_samples.size(); + if (base_total == 0) { + return data; + } + + // Determine output duration from time curve (if any). Time curves are in project-frame domain. + int64_t output_frames = 0; + if (time_keyframe && time_keyframe->GetCount() > 0) { + output_frames = time_keyframe->GetLength(); + } + if (output_frames <= 0) { + // Default to source duration derived from base waveform length + double source_duration = static_cast(base_total) / static_cast(num_per_second); + output_frames = static_cast(std::llround(source_duration * project_fps_value)); + } + double output_duration_seconds = static_cast(output_frames) / project_fps_value; + int total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second)); + + if (total_samples <= 0) { + return data; + } + + data.resize(total_samples); + data.zero(total_samples); + + for (int i = 0; i < total_samples; ++i) { + double out_time = static_cast(i) / static_cast(num_per_second); + // Time keyframes are defined in project-frame domain; evaluate using project frames + double project_frame = out_time * project_fps_value; + double mapped_project_frame = time_keyframe ? time_keyframe->GetValue(project_frame) : project_frame; + // Convert mapped project frame to seconds (project FPS), then to waveform index + double source_time = mapped_project_frame / project_fps_value; + double source_index = source_time * static_cast(num_per_second); + + // Sample base waveform (nearest with simple linear blend) + int idx0 = static_cast(std::floor(source_index)); + int idx1 = idx0 + 1; + double frac = source_index - static_cast(idx0); + + float max_sample = 0.0f; + float rms_sample = 0.0f; + if (idx0 >= 0 && idx0 < static_cast(base_total)) { + max_sample = base.max_samples[idx0]; + rms_sample = base.rms_samples[idx0]; + } + if (idx1 >= 0 && idx1 < static_cast(base_total)) { + max_sample = static_cast((1.0 - frac) * max_sample + frac * base.max_samples[idx1]); + rms_sample = static_cast((1.0 - frac) * rms_sample + frac * base.rms_samples[idx1]); + } + + double gain = 1.0; + if (volume_keyframe) { + double project_frame = out_time * project_fps_value; + gain = volume_keyframe->GetValue(project_frame); + } + max_sample = static_cast(max_sample * gain); + rms_sample = static_cast(rms_sample * gain); + + data.max_samples[i] = max_sample; + data.rms_samples[i] = rms_sample; + } + + if (normalize) { + float samples_max = 0.0f; + for (auto v : data.max_samples) { + samples_max = std::max(samples_max, std::abs(v)); + } + if (samples_max > 0.0f) { + data.scale(total_samples, 1.0f / samples_max); + } + } + + return data; +} + +AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize) { + AudioWaveformData data; + + if (!source_reader || num_per_second <= 0) { + return data; + } + + // Open reader (if needed) + if (!source_reader->IsOpen()) { + source_reader->Open(); + } + + const auto retry_delay = std::chrono::milliseconds(100); + const auto max_wait_for_open = std::chrono::milliseconds(3000); + + auto get_frame_with_retry = [&](int64_t frame_number) -> std::shared_ptr { + std::chrono::steady_clock::time_point wait_start; + bool waiting_for_open = false; + while (true) { + try { + return source_reader->GetFrame(frame_number); + } catch (const openshot::ReaderClosed&) { + auto now = std::chrono::steady_clock::now(); + if (!waiting_for_open) { + waiting_for_open = true; + wait_start = now; + } else if (now - wait_start >= max_wait_for_open) { + throw; + } + + std::this_thread::sleep_for(retry_delay); + } + } + }; + + int sample_rate = source_reader->info.sample_rate; + if (sample_rate <= 0) { + sample_rate = num_per_second; + } + int sample_divisor = sample_rate / num_per_second; + if (sample_divisor <= 0) { + sample_divisor = 1; + } + + // Determine length of video frames (for waveform) + int64_t reader_video_length = source_reader->info.video_length; + if (reader_video_length < 0) { + reader_video_length = 0; + } + float reader_duration = source_reader->info.duration; + double fps_value = source_reader->info.fps.ToDouble(); + float frames_duration = 0.0f; + if (reader_video_length > 0 && fps_value > 0.0) { + frames_duration = static_cast(reader_video_length / fps_value); + } + if (reader_duration <= 0.0f) { + reader_duration = frames_duration; + } + if (reader_duration < 0.0f) { + reader_duration = 0.0f; + } + + if (!source_reader->info.has_audio) { + return data; + } + + int total_samples = static_cast(std::ceil(reader_duration * num_per_second)); + if (total_samples <= 0 || source_reader->info.channels == 0) { + return data; + } + + if (channel != -1 && (channel < 0 || channel >= source_reader->info.channels)) { + return data; + } + + // Resize and clear audio buffers + data.resize(total_samples); + data.zero(total_samples); + + int extracted_index = 0; + int sample_index = 0; + float samples_max = 0.0f; + float chunk_max = 0.0f; + double chunk_squared_sum = 0.0; + + int channel_count = (channel == -1) ? source_reader->info.channels : 1; + std::vector channels(source_reader->info.channels, nullptr); + + try { + for (int64_t f = 1; f <= reader_video_length && extracted_index < total_samples; f++) { + std::shared_ptr frame = get_frame_with_retry(f); + + for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { + if (channel == channel_index || channel == -1) { + channels[channel_index] = frame->GetAudioSamples(channel_index); + } + } + + int sample_count = frame->GetAudioSamplesCount(); + for (int s = 0; s < sample_count; s++) { + for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { + if (channel == channel_index || channel == -1) { + float *samples = channels[channel_index]; + if (!samples) { + continue; + } + float abs_sample = std::abs(samples[s]); + chunk_squared_sum += static_cast(samples[s]) * static_cast(samples[s]); + chunk_max = std::max(chunk_max, abs_sample); + } + } + + sample_index += 1; + + if (sample_index % sample_divisor == 0) { + float avg_squared_sum = 0.0f; + if (channel_count > 0) { + avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_divisor * channel_count)); + } + + if (extracted_index < total_samples) { + data.max_samples[extracted_index] = chunk_max; + data.rms_samples[extracted_index] = std::sqrt(avg_squared_sum); + samples_max = std::max(samples_max, chunk_max); + extracted_index++; + } + + sample_index = 0; + chunk_max = 0.0f; + chunk_squared_sum = 0.0; + + if (extracted_index >= total_samples) { + break; + } + } + } + } + } catch (...) { + throw; + } + + if (sample_index > 0 && extracted_index < total_samples) { + float avg_squared_sum = 0.0f; + if (channel_count > 0) { + avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_index * channel_count)); + } + + data.max_samples[extracted_index] = chunk_max; + data.rms_samples[extracted_index] = std::sqrt(avg_squared_sum); + samples_max = std::max(samples_max, chunk_max); + extracted_index++; + } + + if (normalize && samples_max > 0.0f) { + float scale = 1.0f / samples_max; + data.scale(total_samples, scale); + } + + return data; +} + +ReaderBase* AudioWaveformer::ResolveSourceReader(ReaderBase* source_reader) { + if (!source_reader) { + return nullptr; + } + + ReaderBase* current = source_reader; + while (true) { + if (auto clip = dynamic_cast(current)) { + current = clip->Reader(); + continue; + } + if (auto mapper = dynamic_cast(current)) { + current = mapper->Reader(); + continue; + } + break; + } + return current; +} + +Fraction AudioWaveformer::ResolveSourceFPS(ReaderBase* source_reader) { + if (!source_reader) { + return Fraction(0, 1); + } + return source_reader->info.fps; +} + +// Resolve and cache the reader used for waveform extraction (prefer a detached FFmpegReader clone) +ReaderBase* AudioWaveformer::ResolveWaveformReader() { + if (source_initialized) { + return resolved_reader ? resolved_reader : reader; + } + source_initialized = true; + + resolved_reader = ResolveSourceReader(reader); + + // Prefer a detached, audio-only FFmpegReader clone so we never mutate the live reader used for preview. + if (auto ff_reader = dynamic_cast(resolved_reader)) { + const Json::Value ff_json = ff_reader->JsonValue(); + const std::string path = ff_json.get("path", "").asString(); + if (!path.empty()) { + try { + auto clone = std::make_unique(path, false); + clone->SetJsonValue(ff_json); + clone->info.has_video = false; // explicitly audio-only for waveform extraction + detached_reader = std::move(clone); + resolved_reader = detached_reader.get(); + } catch (...) { + // Fall back to using the original reader if cloning fails + detached_reader.reset(); + resolved_reader = ResolveSourceReader(reader); + } + } + } + + return resolved_reader ? resolved_reader : reader; } diff --git a/src/AudioWaveformer.h b/src/AudioWaveformer.h index 578db4152..73ed9566f 100644 --- a/src/AudioWaveformer.h +++ b/src/AudioWaveformer.h @@ -15,7 +15,11 @@ #include "ReaderBase.h" #include "Frame.h" +#include "KeyFrame.h" +#include "Fraction.h" +#include #include +#include namespace openshot { @@ -79,19 +83,51 @@ namespace openshot { class AudioWaveformer { private: ReaderBase* reader; + std::unique_ptr detached_reader; ///< Optional detached reader clone for waveform extraction + ReaderBase* resolved_reader = nullptr; ///< Cached pointer to the reader used for extraction + bool source_initialized = false; public: /// Default constructor AudioWaveformer(ReaderBase* reader); - /// @brief Extract audio samples from any ReaderBase class + /// @brief Extract audio samples from any ReaderBase class (legacy overload, now delegates to audio-only path) /// @param channel Which audio channel should we extract data from (-1 == all channels) /// @param num_per_second How many samples per second to return /// @param normalize Should we scale the data range so the largest value is 1.0 AudioWaveformData ExtractSamples(int channel, int num_per_second, bool normalize); + /// @brief Extract audio samples from a media file path (audio-only fast path) + AudioWaveformData ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize); + + /// @brief Apply time and volume keyframes to an existing waveform data set + AudioWaveformData ApplyKeyframes(const AudioWaveformData& base, + const openshot::Keyframe* time_keyframe, + const openshot::Keyframe* volume_keyframe, + const openshot::Fraction& project_fps, + const openshot::Fraction& source_fps, + int source_channels, + int num_per_second, + int channel, + bool normalize); + + /// @brief Convenience: extract then apply keyframes in one step from a file path + AudioWaveformData ExtractSamples(const std::string& path, + const openshot::Keyframe* time_keyframe, + const openshot::Keyframe* volume_keyframe, + const openshot::Fraction& project_fps, + int channel, + int num_per_second, + bool normalize); + /// Destructor ~AudioWaveformer(); + + private: + AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize); + openshot::ReaderBase* ResolveSourceReader(openshot::ReaderBase* source_reader); + openshot::Fraction ResolveSourceFPS(openshot::ReaderBase* source_reader); + openshot::ReaderBase* ResolveWaveformReader(); }; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 153b2c1ec..6e22574c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,6 +72,7 @@ set(OPENSHOT_SOURCES Fraction.cpp Frame.cpp FrameMapper.cpp + MemoryTrim.cpp Json.cpp KeyFrame.cpp OpenShotVersion.cpp @@ -114,9 +115,11 @@ set(EFFECTS_SOURCES effects/ColorMap.cpp effects/ColorShift.cpp effects/Crop.cpp + effects/CropHelpers.cpp effects/Deinterlace.cpp effects/Hue.cpp effects/LensFlare.cpp + effects/AnalogTape.cpp effects/Mask.cpp effects/Negate.cpp effects/Pixelate.cpp diff --git a/src/CVTracker.cpp b/src/CVTracker.cpp index 0690f8f14..f243fcfb5 100644 --- a/src/CVTracker.cpp +++ b/src/CVTracker.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include @@ -25,12 +27,22 @@ using namespace openshot; using google::protobuf::util::TimeUtil; +// Clamp a rectangle to image bounds and ensure a minimal size +static inline void clampRect(cv::Rect2d &r, int width, int height) +{ + r.x = std::clamp(r.x, 0.0, double(width - 1)); + r.y = std::clamp(r.y, 0.0, double(height - 1)); + r.width = std::clamp(r.width, 1.0, double(width - r.x)); + r.height = std::clamp(r.height, 1.0, double(height - r.y)); +} + // Constructor CVTracker::CVTracker(std::string processInfoJson, ProcessingController &processingController) : processingController(&processingController), json_interval(false){ SetJson(processInfoJson); start = 1; end = 1; + lostCount = 0; } // Set desirable tracker method @@ -54,152 +66,250 @@ cv::Ptr CVTracker::selectTracker(std::string trackerType){ return nullptr; } -// Track object in the hole clip or in a given interval -void CVTracker::trackClip(openshot::Clip& video, size_t _start, size_t _end, bool process_interval){ - +// Track object in the whole clip or in a given interval +void CVTracker::trackClip(openshot::Clip& video, + size_t _start, + size_t _end, + bool process_interval) +{ video.Open(); - if(!json_interval){ + if (!json_interval) { start = _start; end = _end; - - if(!process_interval || end <= 1 || end-start == 0){ - // Get total number of frames in video - start = (int)(video.Start() * video.Reader()->info.fps.ToFloat()) + 1; - end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1; + if (!process_interval || end <= 1 || end - start == 0) { + start = int(video.Start() * video.Reader()->info.fps.ToFloat()) + 1; + end = int(video.End() * video.Reader()->info.fps.ToFloat()) + 1; } + } else { + start = int(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1; + end = int(video.End() * video.Reader()->info.fps.ToFloat()) + 1; } - else{ - start = (int)(start + video.Start() * video.Reader()->info.fps.ToFloat()) + 1; - end = (int)(video.End() * video.Reader()->info.fps.ToFloat()) + 1; - } - - if(error){ - return; - } - + if (error) return; processingController->SetError(false, ""); - bool trackerInit = false; - - size_t frame; - // Loop through video - for (frame = start; frame <= end; frame++) - { - // Stop the feature tracker process - if(processingController->ShouldStop()){ - return; - } + bool trackerInit = false; + lostCount = 0; // reset lost counter once at the start - size_t frame_number = frame; - // Get current frame - std::shared_ptr f = video.GetFrame(frame_number); + for (size_t frame = start; frame <= end; ++frame) { + if (processingController->ShouldStop()) return; - // Grab OpenCV Mat image - cv::Mat cvimage = f->GetImageCV(); + auto f = video.GetFrame(frame); + cv::Mat img = f->GetImageCV(); - if(frame == start){ - // Take the normalized inital bounding box and multiply to the current video shape - bbox = cv::Rect2d(int(bbox.x*cvimage.cols), int(bbox.y*cvimage.rows), - int(bbox.width*cvimage.cols), int(bbox.height*cvimage.rows)); + if (frame == start) { + bbox = cv::Rect2d( + int(bbox.x * img.cols), + int(bbox.y * img.rows), + int(bbox.width * img.cols), + int(bbox.height * img.rows) + ); } - // Pass the first frame to initialize the tracker - if(!trackerInit){ - - // Initialize the tracker - initTracker(cvimage, frame_number); - + if (!trackerInit) { + initTracker(img, frame); trackerInit = true; + lostCount = 0; } - else{ - // Update the object tracker according to frame - trackerInit = trackFrame(cvimage, frame_number); - - // Draw box on image - FrameData fd = GetTrackedData(frame_number); + else { + // trackFrame now manages lostCount and will re-init internally + trackFrame(img, frame); + // record whatever bbox we have now + FrameData fd = GetTrackedData(frame); } - // Update progress - processingController->SetProgress(uint(100*(frame_number-start)/(end-start))); + + processingController->SetProgress( + uint(100 * (frame - start) / (end - start)) + ); } } // Initialize the tracker -bool CVTracker::initTracker(cv::Mat &frame, size_t frameId){ - +bool CVTracker::initTracker(cv::Mat &frame, size_t frameId) +{ // Create new tracker object tracker = selectTracker(trackerType); - // Correct if bounding box contains negative proportions (width and/or height < 0) - if(bbox.width < 0){ - bbox.x = bbox.x - abs(bbox.width); - bbox.width = abs(bbox.width); + // Correct negative width/height + if (bbox.width < 0) { + bbox.x -= bbox.width; + bbox.width = -bbox.width; } - if(bbox.height < 0){ - bbox.y = bbox.y - abs(bbox.height); - bbox.height = abs(bbox.height); + if (bbox.height < 0) { + bbox.y -= bbox.height; + bbox.height = -bbox.height; } + // Clamp to frame bounds + clampRect(bbox, frame.cols, frame.rows); + // Initialize tracker tracker->init(frame, bbox); - float fw = frame.size().width; - float fh = frame.size().height; + float fw = float(frame.cols), fh = float(frame.rows); + + // record original pixel size + origWidth = bbox.width; + origHeight = bbox.height; + + // initialize sub-pixel smoother at true center + smoothC_x = bbox.x + bbox.width * 0.5; + smoothC_y = bbox.y + bbox.height * 0.5; // Add new frame data - trackedDataById[frameId] = FrameData(frameId, 0, (bbox.x)/fw, - (bbox.y)/fh, - (bbox.x+bbox.width)/fw, - (bbox.y+bbox.height)/fh); + trackedDataById[frameId] = FrameData( + frameId, 0, + bbox.x / fw, + bbox.y / fh, + (bbox.x + bbox.width) / fw, + (bbox.y + bbox.height) / fh + ); return true; } // Update the object tracker according to frame -bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId){ - // Update the tracking result - bool ok = tracker->update(frame, bbox); +// returns true if KLT succeeded, false otherwise +bool CVTracker::trackFrame(cv::Mat &frame, size_t frameId) +{ + const int W = frame.cols, H = frame.rows; + const auto& prev = trackedDataById[frameId - 1]; + + // Reconstruct last-known box in pixel coords + cv::Rect2d lastBox( + prev.x1 * W, prev.y1 * H, + (prev.x2 - prev.x1) * W, + (prev.y2 - prev.y1) * H + ); + + // Convert to grayscale + cv::Mat gray; + cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY); + + cv::Rect2d cand; + bool didKLT = false; + + // Try KLT-based drift + if (!prevGray.empty() && !prevPts.empty()) { + std::vector currPts; + std::vector status; + std::vector err; + cv::calcOpticalFlowPyrLK( + prevGray, gray, + prevPts, currPts, + status, err, + cv::Size(21,21), 3, + cv::TermCriteria{cv::TermCriteria::COUNT|cv::TermCriteria::EPS,30,0.01}, + cv::OPTFLOW_LK_GET_MIN_EIGENVALS, 1e-4 + ); + + // collect per-point displacements + std::vector dx, dy; + for (size_t i = 0; i < status.size(); ++i) { + if (status[i] && err[i] < 12.0) { + dx.push_back(currPts[i].x - prevPts[i].x); + dy.push_back(currPts[i].y - prevPts[i].y); + } + } - // Add frame number and box coords if tracker finds the object - // Otherwise add only frame number - if (ok) - { - float fw = frame.size().width; - float fh = frame.size().height; - - cv::Rect2d filtered_box = filter_box_jitter(frameId); - // Add new frame data - trackedDataById[frameId] = FrameData(frameId, 0, (filtered_box.x)/fw, - (filtered_box.y)/fh, - (filtered_box.x+filtered_box.width)/fw, - (filtered_box.y+filtered_box.height)/fh); - } - else - { - // Copy the last frame data if the tracker get lost - trackedDataById[frameId] = trackedDataById[frameId-1]; + if ((int)dx.size() >= minKltPts) { + auto median = [&](auto &v){ + std::nth_element(v.begin(), v.begin()+v.size()/2, v.end()); + return v[v.size()/2]; + }; + double mdx = median(dx), mdy = median(dy); + + cand = lastBox; + cand.x += mdx; + cand.y += mdy; + cand.width = origWidth; + cand.height = origHeight; + + lostCount = 0; + didKLT = true; + } } - return ok; -} + // Fallback to whole-frame flow if KLT failed + if (!didKLT) { + ++lostCount; + cand = lastBox; + if (!fullPrevGray.empty()) { + cv::Mat flow; + cv::calcOpticalFlowFarneback( + fullPrevGray, gray, flow, + 0.5,3,15,3,5,1.2,0 + ); + cv::Scalar avg = cv::mean(flow); + cand.x += avg[0]; + cand.y += avg[1]; + } + cand.width = origWidth; + cand.height = origHeight; -cv::Rect2d CVTracker::filter_box_jitter(size_t frameId){ - // get tracked data for the previous frame - float last_box_width = trackedDataById[frameId-1].x2 - trackedDataById[frameId-1].x1; - float last_box_height = trackedDataById[frameId-1].y2 - trackedDataById[frameId-1].y1; + if (lostCount >= 10) { + initTracker(frame, frameId); + cand = bbox; + lostCount = 0; + } + } - float curr_box_width = bbox.width; - float curr_box_height = bbox.height; - // keep the last width and height if the difference is less than 1% - float threshold = 0.01; + // Dead-zone sub-pixel smoothing + { + constexpr double JITTER_THRESH = 1.0; + double measCx = cand.x + cand.width * 0.5; + double measCy = cand.y + cand.height * 0.5; + double dx = measCx - smoothC_x; + double dy = measCy - smoothC_y; + + if (std::abs(dx) > JITTER_THRESH || std::abs(dy) > JITTER_THRESH) { + smoothC_x = measCx; + smoothC_y = measCy; + } - cv::Rect2d filtered_box = bbox; - if(std::abs(1-(curr_box_width/last_box_width)) <= threshold){ - filtered_box.width = last_box_width; + cand.x = smoothC_x - cand.width * 0.5; + cand.y = smoothC_y - cand.height * 0.5; } - if(std::abs(1-(curr_box_height/last_box_height)) <= threshold){ - filtered_box.height = last_box_height; + + + // Candidate box may now lie outside frame; ROI for KLT is clamped below + // Re-seed KLT features + { + // Clamp ROI to frame bounds and avoid negative width/height + int roiX = int(std::clamp(cand.x, 0.0, double(W - 1))); + int roiY = int(std::clamp(cand.y, 0.0, double(H - 1))); + int roiW = int(std::min(cand.width, double(W - roiX))); + int roiH = int(std::min(cand.height, double(H - roiY))); + roiW = std::max(0, roiW); + roiH = std::max(0, roiH); + + if (roiW > 0 && roiH > 0) { + cv::Rect roi(roiX, roiY, roiW, roiH); + cv::goodFeaturesToTrack( + gray(roi), prevPts, + kltMaxCorners, kltQualityLevel, + kltMinDist, cv::Mat(), kltBlockSize + ); + for (auto &pt : prevPts) + pt += cv::Point2f(float(roi.x), float(roi.y)); + } else { + prevPts.clear(); + } } - return filtered_box; + + // Commit state + fullPrevGray = gray.clone(); + prevGray = gray.clone(); + bbox = cand; + float fw = float(W), fh = float(H); + trackedDataById[frameId] = FrameData( + frameId, 0, + cand.x / fw, + cand.y / fh, + (cand.x + cand.width) / fw, + (cand.y + cand.height) / fh + ); + + return didKLT; } bool CVTracker::SaveTrackedData(){ diff --git a/src/CVTracker.h b/src/CVTracker.h index eff8b50a8..023d9297b 100644 --- a/src/CVTracker.h +++ b/src/CVTracker.h @@ -94,11 +94,25 @@ namespace openshot bool error = false; - // Initialize the tracker - bool initTracker(cv::Mat &frame, size_t frameId); - - // Update the object tracker according to frame - bool trackFrame(cv::Mat &frame, size_t frameId); + // count of consecutive “missed” frames + int lostCount{0}; + + // KLT parameters and state + cv::Mat prevGray; // last frame in gray + std::vector prevPts; // tracked feature points + const int kltMaxCorners = 100; // max features to keep + const double kltQualityLevel = 0.01; // goodFeatures threshold + const double kltMinDist = 5.0; // min separation + const int kltBlockSize = 3; // window for feature detect + const int minKltPts = 10; // below this, we assume occluded + double smoothC_x = 0, smoothC_y = 0; ///< running, sub-pixel center + const double smoothAlpha = 0.8; ///< [0..1], higher → tighter but more jitter + + // full-frame fall-back + cv::Mat fullPrevGray; + + // last known good box size + double origWidth{0}, origHeight{0}; public: @@ -113,8 +127,11 @@ namespace openshot /// If start, end and process_interval are passed as argument, clip will be processed in [start,end) void trackClip(openshot::Clip& video, size_t _start=0, size_t _end=0, bool process_interval=false); - /// Filter current bounding box jitter - cv::Rect2d filter_box_jitter(size_t frameId); + // Update the object tracker according to frame + bool trackFrame(cv::Mat &frame, size_t frameId); + + // Initialize the tracker + bool initTracker(cv::Mat &frame, size_t frameId); /// Get tracked data for a given frame FrameData GetTrackedData(size_t frameId); diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp index b768c12f2..bd9a6c2c2 100644 --- a/src/CacheMemory.cpp +++ b/src/CacheMemory.cpp @@ -13,12 +13,13 @@ #include "CacheMemory.h" #include "Exceptions.h" #include "Frame.h" +#include "MemoryTrim.h" using namespace std; using namespace openshot; // Default constructor, no max bytes -CacheMemory::CacheMemory() : CacheBase(0) { +CacheMemory::CacheMemory() : CacheBase(0), bytes_freed_since_trim(0) { // Set cache type name cache_type = "CacheMemory"; range_version = 0; @@ -26,7 +27,7 @@ CacheMemory::CacheMemory() : CacheBase(0) { } // Constructor that sets the max bytes to cache -CacheMemory::CacheMemory(int64_t max_bytes) : CacheBase(max_bytes) { +CacheMemory::CacheMemory(int64_t max_bytes) : CacheBase(max_bytes), bytes_freed_since_trim(0) { // Set cache type name cache_type = "CacheMemory"; range_version = 0; @@ -161,6 +162,7 @@ void CacheMemory::Remove(int64_t start_frame_number, int64_t end_frame_number) { // Create a scoped lock, to protect the cache from multiple threads const std::lock_guard lock(*cacheMutex); + int64_t removed_bytes = 0; // Loop through frame numbers std::deque::iterator itr; @@ -180,6 +182,10 @@ void CacheMemory::Remove(int64_t start_frame_number, int64_t end_frame_number) { if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number) { + // Count bytes freed before erasing the frame + if (frames.count(*itr_ordered)) + removed_bytes += frames[*itr_ordered]->GetBytes(); + // erase frame number frames.erase(*itr_ordered); itr_ordered = ordered_frame_numbers.erase(itr_ordered); @@ -187,6 +193,17 @@ void CacheMemory::Remove(int64_t start_frame_number, int64_t end_frame_number) itr_ordered++; } + if (removed_bytes > 0) + { + bytes_freed_since_trim += removed_bytes; + if (bytes_freed_since_trim >= TRIM_THRESHOLD_BYTES) + { + // Periodically return freed arenas to the OS + if (TrimMemoryToOS()) + bytes_freed_since_trim = 0; + } + } + // Needs range processing (since cache has changed) needs_range_processing = true; } @@ -229,6 +246,10 @@ void CacheMemory::Clear() ordered_frame_numbers.clear(); ordered_frame_numbers.shrink_to_fit(); needs_range_processing = true; + bytes_freed_since_trim = 0; + + // Trim freed arenas back to OS after large clears + TrimMemoryToOS(true); } // Count the frames in the queue diff --git a/src/CacheMemory.h b/src/CacheMemory.h index e35fdb11c..9972b1025 100644 --- a/src/CacheMemory.h +++ b/src/CacheMemory.h @@ -28,8 +28,10 @@ namespace openshot { */ class CacheMemory : public CacheBase { private: + static constexpr int64_t TRIM_THRESHOLD_BYTES = 1024LL * 1024 * 1024; ///< Release memory after freeing this much memory std::map > frames; ///< This map holds the frame number and Frame objects std::deque frame_numbers; ///< This queue holds a sequential list of cached Frame numbers + int64_t bytes_freed_since_trim; ///< Tracks bytes freed to trigger a heap trim /// Clean up cached frames that exceed the max number of bytes void CleanUp(); diff --git a/src/Clip.cpp b/src/Clip.cpp index b63fdba7c..b385ec538 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -22,6 +22,9 @@ #include "Timeline.h" #include "ZmqLogger.h" +#include +#include + #ifdef USE_IMAGEMAGICK #include "MagickUtilities.h" #include "ImageReader.h" @@ -32,6 +35,34 @@ using namespace openshot; +namespace { + struct CompositeChoice { const char* name; CompositeType value; }; + const CompositeChoice composite_choices[] = { + {"Normal", COMPOSITE_SOURCE_OVER}, + + // Darken group + {"Darken", COMPOSITE_DARKEN}, + {"Multiply", COMPOSITE_MULTIPLY}, + {"Color Burn", COMPOSITE_COLOR_BURN}, + + // Lighten group + {"Lighten", COMPOSITE_LIGHTEN}, + {"Screen", COMPOSITE_SCREEN}, + {"Color Dodge", COMPOSITE_COLOR_DODGE}, + {"Add", COMPOSITE_PLUS}, + + // Contrast group + {"Overlay", COMPOSITE_OVERLAY}, + {"Soft Light", COMPOSITE_SOFT_LIGHT}, + {"Hard Light", COMPOSITE_HARD_LIGHT}, + + // Compare + {"Difference", COMPOSITE_DIFFERENCE}, + {"Exclusion", COMPOSITE_EXCLUSION}, + }; + const int composite_choices_count = sizeof(composite_choices)/sizeof(CompositeChoice); +} + // Init default settings for a clip void Clip::init_settings() { @@ -45,6 +76,7 @@ void Clip::init_settings() anchor = ANCHOR_CANVAS; display = FRAME_DISPLAY_NONE; mixing = VOLUME_MIX_NONE; + composite = COMPOSITE_SOURCE_OVER; waveform = false; previous_properties = ""; parentObjectId = ""; @@ -112,36 +144,51 @@ void Clip::init_reader_settings() { } void Clip::init_reader_rotation() { - // Don't init rotation if clip already has keyframes. - if (rotation.GetCount() > 0) + // Only apply metadata rotation if clip rotation has not been explicitly set. + if (rotation.GetCount() > 0 || !reader) return; - // Get rotation from metadata (if any) + const auto rotate_meta = reader->info.metadata.find("rotate"); + if (rotate_meta == reader->info.metadata.end()) { + // Ensure rotation keyframes always start with a default 0° point. + rotation = Keyframe(0.0f); + return; + } + float rotate_angle = 0.0f; - if (reader && reader->info.metadata.count("rotate") > 0) { - try { - rotate_angle = strtof(reader->info.metadata["rotate"].c_str(), nullptr); - } catch (const std::exception& e) { - // Leave rotate_angle at default 0.0f - } + try { + rotate_angle = strtof(rotate_meta->second.c_str(), nullptr); + } catch (const std::exception& e) { + return; // ignore invalid metadata } + rotation = Keyframe(rotate_angle); - // Compute uniform scale factors for rotated video. - // Assume reader->info.width and reader->info.height are the clip's natural dimensions. + // Do not overwrite user-authored scale curves. + auto has_default_scale = [](const Keyframe& kf) { + return kf.GetCount() == 1 && fabs(kf.GetPoint(0).co.Y - 1.0) < 0.00001; + }; + if (!has_default_scale(scale_x) || !has_default_scale(scale_y)) + return; + + // No need to adjust scaling when the metadata rotation is effectively zero. + if (fabs(rotate_angle) < 0.0001f) + return; + float w = static_cast(reader->info.width); float h = static_cast(reader->info.height); - float rad = rotate_angle * M_PI / 180.0f; + if (w <= 0.0f || h <= 0.0f) + return; + + float rad = rotate_angle * static_cast(M_PI) / 180.0f; - // Calculate the dimensions of the bounding box for the rotated clip. float new_width = fabs(w * cos(rad)) + fabs(h * sin(rad)); float new_height = fabs(w * sin(rad)) + fabs(h * cos(rad)); + if (new_width <= 0.0f || new_height <= 0.0f) + return; - // To have the rotated clip appear the same size as the unrotated clip, - // compute a uniform scale factor S that brings the bounding box back to (w, h). float uniform_scale = std::min(w / new_width, h / new_height); - // Set scale keyframes uniformly. scale_x = Keyframe(uniform_scale); scale_y = Keyframe(uniform_scale); } @@ -515,8 +562,13 @@ std::shared_ptr Clip::GetParentTrackedObject() { // Get file extension std::string Clip::get_file_extension(std::string path) { - // return last part of path - return path.substr(path.find_last_of(".") + 1); + // Return last part of path safely (handle filenames without a dot) + const auto dot_pos = path.find_last_of('.'); + if (dot_pos == std::string::npos || dot_pos + 1 >= path.size()) { + return std::string(); + } + + return path.substr(dot_pos + 1); } // Adjust the audio and image of a time mapped frame @@ -540,7 +592,8 @@ void Clip::apply_timemapping(std::shared_ptr frame) // Get delta (difference from this frame to the next time mapped frame: Y value) double delta = time.GetDelta(clip_frame_number + 1); - bool is_increasing = time.IsIncreasing(clip_frame_number + 1); + const bool prev_is_increasing = time.IsIncreasing(clip_frame_number); + const bool is_increasing = time.IsIncreasing(clip_frame_number + 1); // Determine length of source audio (in samples) // A delta of 1.0 == normal expected samples @@ -553,7 +606,7 @@ void Clip::apply_timemapping(std::shared_ptr frame) // Determine starting audio location AudioLocation location; - if (previous_location.frame == 0 || abs(new_frame_number - previous_location.frame) > 2) { + if (previous_location.frame == 0 || abs(new_frame_number - previous_location.frame) > 2 || prev_is_increasing != is_increasing) { // No previous location OR gap detected location.frame = new_frame_number; location.sample_start = 0; @@ -562,6 +615,7 @@ void Clip::apply_timemapping(std::shared_ptr frame) // We don't want to interpolate between unrelated audio data if (resampler) { delete resampler; + resampler = nullptr; } // Init resampler with # channels from Reader (should match the timeline) resampler = new AudioResampler(Reader()->info.channels); @@ -595,6 +649,12 @@ void Clip::apply_timemapping(std::shared_ptr frame) std::shared_ptr source_frame = GetOrCreateFrame(location.frame, false); int frame_sample_count = source_frame->GetAudioSamplesCount() - location.sample_start; + // Inform FrameMapper of the direction for THIS mapper frame + if (auto *fm = dynamic_cast(reader)) { + fm->SetDirectionHint(is_increasing); + } + source_frame->SetAudioDirection(is_increasing); + if (frame_sample_count == 0) { // No samples found in source frame (fill with silence) if (is_increasing) { @@ -681,10 +741,17 @@ std::shared_ptr Clip::GetOrCreateFrame(int64_t number, bool enable_time) try { // Init to requested frame int64_t clip_frame_number = adjust_frame_number_minimum(number); + bool is_increasing = true; // Adjust for time-mapping (if any) if (enable_time && time.GetLength() > 1) { - clip_frame_number = adjust_frame_number_minimum(time.GetLong(clip_frame_number)); + is_increasing = time.IsIncreasing(clip_frame_number + 1); + const int64_t time_frame_number = adjust_frame_number_minimum(time.GetLong(clip_frame_number)); + if (auto *fm = dynamic_cast(reader)) { + // Inform FrameMapper which direction this mapper frame is being requested + fm->SetDirectionHint(is_increasing); + } + clip_frame_number = time_frame_number; } // Debug output @@ -694,10 +761,12 @@ std::shared_ptr Clip::GetOrCreateFrame(int64_t number, bool enable_time) // Attempt to get a frame (but this could fail if a reader has just been closed) auto reader_frame = reader->GetFrame(clip_frame_number); - reader_frame->number = number; // Override frame # (due to time-mapping might change it) - - // Return real frame if (reader_frame) { + // Override frame # (due to time-mapping might change it) + reader_frame->number = number; + reader_frame->SetAudioDirection(is_increasing); + + // Return real frame // Create a new copy of reader frame // This allows a clip to modify the pixels and audio of this frame without // changing the underlying reader's frame data @@ -760,6 +829,7 @@ std::string Clip::PropertiesJSON(int64_t requested_frame) const { root["scale"] = add_property_json("Scale", scale, "int", "", NULL, 0, 3, false, requested_frame); root["display"] = add_property_json("Frame Number", display, "int", "", NULL, 0, 3, false, requested_frame); root["mixing"] = add_property_json("Volume Mixing", mixing, "int", "", NULL, 0, 2, false, requested_frame); + root["composite"] = add_property_json("Composite", composite, "int", "", NULL, 0, composite_choices_count - 1, false, requested_frame); root["waveform"] = add_property_json("Waveform", waveform, "int", "", NULL, 0, 1, false, requested_frame); root["parentObjectId"] = add_property_json("Parent", 0.0, "string", parentObjectId, NULL, -1, -1, false, requested_frame); @@ -791,6 +861,10 @@ std::string Clip::PropertiesJSON(int64_t requested_frame) const { root["mixing"]["choices"].append(add_property_choice_json("Average", VOLUME_MIX_AVERAGE, mixing)); root["mixing"]["choices"].append(add_property_choice_json("Reduce", VOLUME_MIX_REDUCE, mixing)); + // Add composite choices (dropdown style) + for (int i = 0; i < composite_choices_count; ++i) + root["composite"]["choices"].append(add_property_choice_json(composite_choices[i].name, composite_choices[i].value, composite)); + // Add waveform choices (dropdown style) root["waveform"]["choices"].append(add_property_choice_json("Yes", true, waveform)); root["waveform"]["choices"].append(add_property_choice_json("No", false, waveform)); @@ -873,6 +947,7 @@ Json::Value Clip::JsonValue() const { root["anchor"] = anchor; root["display"] = display; root["mixing"] = mixing; + root["composite"] = composite; root["waveform"] = waveform; root["scale_x"] = scale_x.JsonValue(); root["scale_y"] = scale_y.JsonValue(); @@ -961,6 +1036,8 @@ void Clip::SetJsonValue(const Json::Value root) { display = (FrameDisplayType) root["display"].asInt(); if (!root["mixing"].isNull()) mixing = (VolumeMixType) root["mixing"].asInt(); + if (!root["composite"].isNull()) + composite = (CompositeType) root["composite"].asInt(); if (!root["waveform"].isNull()) waveform = root["waveform"].asBool(); if (!root["scale_x"].isNull()) @@ -1189,10 +1266,9 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); - painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true); // Composite a new layer onto the image - painter.setCompositionMode(QPainter::CompositionMode_SourceOver); + painter.setCompositionMode(static_cast(composite)); painter.drawImage(0, 0, *frame->GetImage()); painter.end(); @@ -1247,14 +1323,26 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { // Load timeline's new frame image into a QPainter QPainter painter(background_canvas.get()); - painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true); - + painter.setRenderHint(QPainter::TextAntialiasing, true); + if (!transform.isIdentity()) { + painter.setRenderHint(QPainter::SmoothPixmapTransform, true); + } // Apply transform (translate, rotate, scale) painter.setTransform(transform); // Composite a new layer onto the image - painter.setCompositionMode(QPainter::CompositionMode_SourceOver); - painter.drawImage(0, 0, *source_image); + painter.setCompositionMode(static_cast(composite)); + + // Apply opacity via painter instead of per-pixel alpha manipulation + const float alpha_value = alpha.GetValue(frame->number); + if (alpha_value != 1.0f) { + painter.setOpacity(alpha_value); + painter.drawImage(0, 0, *source_image); + // Reset so any subsequent drawing (e.g., overlays) isn’t faded + painter.setOpacity(1.0); + } else { + painter.drawImage(0, 0, *source_image); + } if (timeline) { Timeline *t = static_cast(timeline); @@ -1347,31 +1435,6 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig // Get image from clip std::shared_ptr source_image = frame->GetImage(); - /* ALPHA & OPACITY */ - if (alpha.GetValue(frame->number) != 1.0) - { - float alpha_value = alpha.GetValue(frame->number); - - // Get source image's pixels - unsigned char *pixels = source_image->bits(); - - // Loop through pixels - for (int pixel = 0, byte_index=0; pixel < source_image->width() * source_image->height(); pixel++, byte_index+=4) - { - // Apply alpha to pixel values (since we use a premultiplied value, we must - // multiply the alpha with all colors). - pixels[byte_index + 0] *= alpha_value; - pixels[byte_index + 1] *= alpha_value; - pixels[byte_index + 2] *= alpha_value; - pixels[byte_index + 3] *= alpha_value; - } - - // Debug output - ZmqLogger::Instance()->AppendDebugMethod("Clip::get_transform (Set Alpha & Opacity)", - "alpha_value", alpha_value, - "frame->number", frame->number); - } - /* RESIZE SOURCE IMAGE - based on scale type */ QSize source_size = scale_size(source_image->size(), scale, width, height); diff --git a/src/Clip.h b/src/Clip.h index caeabd57b..cfb37768a 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -151,6 +151,15 @@ namespace openshot { /// Get a frame object or create a blank one std::shared_ptr GetOrCreateFrame(int64_t number, bool enable_time=true); + /// Determine the frames-per-second context used for timeline playback + double resolve_timeline_fps() const; + + /// Determine the number of frames implied by time-mapping curves + int64_t curve_extent_frames() const; + + /// Determine the number of frames implied by the clip's trim range + int64_t trim_extent_frames(double fps_value) const; + /// Adjust the audio and image of a time mapped frame void apply_timemapping(std::shared_ptr frame); @@ -169,6 +178,7 @@ namespace openshot { openshot::AnchorType anchor; ///< The anchor determines what parent a clip should snap to openshot::FrameDisplayType display; ///< The format to display the frame number (if any) openshot::VolumeMixType mixing; ///< What strategy should be followed when mixing audio with other clips + openshot::CompositeType composite; ///< How this clip is composited onto lower layers #ifdef USE_OPENCV bool COMPILED_WITH_CV = true; diff --git a/src/ClipBase.h b/src/ClipBase.h index 4ba6b2f47..732160c16 100644 --- a/src/ClipBase.h +++ b/src/ClipBase.h @@ -16,8 +16,8 @@ #include #include "CacheMemory.h" #include "Frame.h" -#include "Point.h" #include "KeyFrame.h" +#include "IdGenerator.h" #include "Json.h" #include "TimelineBase.h" @@ -48,6 +48,7 @@ namespace openshot { public: /// Constructor for the base clip ClipBase() : + id(IdGenerator::Generate()), position(0.0), layer(0), start(0.0), diff --git a/src/EffectInfo.cpp b/src/EffectInfo.cpp index a9f67028d..dbedf534f 100644 --- a/src/EffectInfo.cpp +++ b/src/EffectInfo.cpp @@ -12,6 +12,7 @@ #include "EffectInfo.h" #include "Effects.h" +#include "effects/AnalogTape.h" using namespace openshot; @@ -25,6 +26,9 @@ std::string EffectInfo::Json() { // Create a new effect instance EffectBase* EffectInfo::CreateEffect(std::string effect_type) { // Init the matching effect object + if (effect_type == "AnalogTape") + return new AnalogTape(); + if (effect_type == "Bars") return new Bars(); @@ -133,6 +137,7 @@ Json::Value EffectInfo::JsonValue() { Json::Value root; // Append info JSON from each supported effect + root.append(AnalogTape().JsonInfo()); root.append(Bars().JsonInfo()); root.append(Blur().JsonInfo()); root.append(Brightness().JsonInfo()); diff --git a/src/Effects.h b/src/Effects.h index ad577f32a..c69776f9d 100644 --- a/src/Effects.h +++ b/src/Effects.h @@ -14,6 +14,7 @@ // SPDX-License-Identifier: LGPL-3.0-or-later /* Effects */ +#include "effects/AnalogTape.h" #include "effects/Bars.h" #include "effects/Blur.h" #include "effects/Brightness.h" diff --git a/src/Enums.h b/src/Enums.h index 14b693166..cea6a1a24 100644 --- a/src/Enums.h +++ b/src/Enums.h @@ -56,6 +56,13 @@ enum FrameDisplayType FRAME_DISPLAY_BOTH ///< Display both the clip's and timeline's frame number }; +/// This enumeration determines which duration source to favor. +enum class DurationStrategy { + LongestStream, ///< Use the longest value from video, audio, or container + VideoPreferred, ///< Prefer the video stream's duration, fallback to audio then container + AudioPreferred, ///< Prefer the audio stream's duration, fallback to video then container +}; + /// This enumeration determines the strategy when mixing audio with other clips. enum VolumeMixType { @@ -64,6 +71,37 @@ enum VolumeMixType VOLUME_MIX_REDUCE ///< Reduce volume by about %25, and then mix (louder, but could cause pops if the sum exceeds 100%) }; +/// This enumeration determines how clips are composited onto lower layers. +enum CompositeType { + COMPOSITE_SOURCE_OVER, + COMPOSITE_DESTINATION_OVER, + COMPOSITE_CLEAR, + COMPOSITE_SOURCE, + COMPOSITE_DESTINATION, + COMPOSITE_SOURCE_IN, + COMPOSITE_DESTINATION_IN, + COMPOSITE_SOURCE_OUT, + COMPOSITE_DESTINATION_OUT, + COMPOSITE_SOURCE_ATOP, + COMPOSITE_DESTINATION_ATOP, + COMPOSITE_XOR, + + // svg 1.2 blend modes + COMPOSITE_PLUS, + COMPOSITE_MULTIPLY, + COMPOSITE_SCREEN, + COMPOSITE_OVERLAY, + COMPOSITE_DARKEN, + COMPOSITE_LIGHTEN, + COMPOSITE_COLOR_DODGE, + COMPOSITE_COLOR_BURN, + COMPOSITE_HARD_LIGHT, + COMPOSITE_SOFT_LIGHT, + COMPOSITE_DIFFERENCE, + COMPOSITE_EXCLUSION, + + COMPOSITE_LAST = COMPOSITE_EXCLUSION +}; /// This enumeration determines the distortion type of Distortion Effect. enum DistortionType diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index d66749bb7..8e38d48d1 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -15,12 +15,16 @@ #include // for std::this_thread::sleep_for #include // for std::chrono::milliseconds +#include +#include #include #include "FFmpegUtilities.h" +#include "effects/CropHelpers.h" #include "FFmpegReader.h" #include "Exceptions.h" +#include "MemoryTrim.h" #include "Timeline.h" #include "ZmqLogger.h" @@ -69,11 +73,14 @@ int hw_de_on = 0; #endif FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) + : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {} + +FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader) : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), NO_PTS_OFFSET(-99999), path(path), is_video_seek(true), check_interlace(false), check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0),is_duration_known(false), largest_frame_processed(0), - current_video_frame(0), packet(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS), audio_pts(0), - video_pts(0), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL), + current_video_frame(0), packet(NULL), duration_strategy(duration_strategy), + audio_pts(0), video_pts(0), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL), pStream(NULL), aStream(NULL), pFrame(NULL), previous_packet_location{-1,0}, hold_packet(false) { @@ -87,8 +94,8 @@ FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) audio_pts_seconds = NO_PTS_OFFSET; // Init cache - working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels); - final_cache.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels); + working_cache.SetMaxBytesFromInfo(info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels); + final_cache.SetMaxBytesFromInfo(24, info.width, info.height, info.sample_rate, info.channels); // Open and Close the reader, to populate its attributes (such as height, width, etc...) if (inspect_reader) { @@ -605,8 +612,8 @@ void FFmpegReader::Open() { previous_packet_location.sample_start = 0; // Adjust cache size based on size of frame and audio - working_cache.SetMaxBytesFromInfo(max_concurrent_frames * info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels); - final_cache.SetMaxBytesFromInfo(max_concurrent_frames * 2, info.width, info.height, info.sample_rate, info.channels); + working_cache.SetMaxBytesFromInfo(info.fps.ToDouble() * 2, info.width, info.height, info.sample_rate, info.channels); + final_cache.SetMaxBytesFromInfo(24, info.width, info.height, info.sample_rate, info.channels); // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp. // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero. @@ -708,6 +715,9 @@ void FFmpegReader::Close() { avformat_close_input(&pFormatCtx); av_freep(&pFormatCtx); + // Release free’d arenas back to OS after heavy teardown + TrimMemoryToOS(true); + // Reset some variables last_frame = 0; hold_packet = false; @@ -727,6 +737,74 @@ bool FFmpegReader::HasAlbumArt() { && (pFormatCtx->streams[videoStream]->disposition & AV_DISPOSITION_ATTACHED_PIC); } +double FFmpegReader::PickDurationSeconds() const { + auto has_value = [](double value) { return value > 0.0; }; + + switch (duration_strategy) { + case DurationStrategy::VideoPreferred: + if (has_value(video_stream_duration_seconds)) + return video_stream_duration_seconds; + if (has_value(audio_stream_duration_seconds)) + return audio_stream_duration_seconds; + if (has_value(format_duration_seconds)) + return format_duration_seconds; + break; + case DurationStrategy::AudioPreferred: + if (has_value(audio_stream_duration_seconds)) + return audio_stream_duration_seconds; + if (has_value(video_stream_duration_seconds)) + return video_stream_duration_seconds; + if (has_value(format_duration_seconds)) + return format_duration_seconds; + break; + case DurationStrategy::LongestStream: + default: + { + double longest = 0.0; + if (has_value(video_stream_duration_seconds)) + longest = std::max(longest, video_stream_duration_seconds); + if (has_value(audio_stream_duration_seconds)) + longest = std::max(longest, audio_stream_duration_seconds); + if (has_value(format_duration_seconds)) + longest = std::max(longest, format_duration_seconds); + if (has_value(longest)) + return longest; + } + break; + } + + if (has_value(format_duration_seconds)) + return format_duration_seconds; + if (has_value(inferred_duration_seconds)) + return inferred_duration_seconds; + + return 0.0; +} + +void FFmpegReader::ApplyDurationStrategy() { + const double fps_value = info.fps.ToDouble(); + const double chosen_seconds = PickDurationSeconds(); + + if (chosen_seconds <= 0.0 || fps_value <= 0.0) { + info.duration = 0.0f; + info.video_length = 0; + is_duration_known = false; + return; + } + + const int64_t frames = static_cast(std::llround(chosen_seconds * fps_value)); + if (frames <= 0) { + info.duration = 0.0f; + info.video_length = 0; + is_duration_known = false; + return; + } + + info.video_length = frames; + info.duration = static_cast(static_cast(frames) / fps_value); + is_duration_known = true; +} + void FFmpegReader::UpdateAudioInfo() { // Set default audio channel layout (if needed) #if HAVE_CH_LAYOUT @@ -742,6 +820,11 @@ void FFmpegReader::UpdateAudioInfo() { return; } + auto record_duration = [](double &target, double seconds) { + if (seconds > 0.0) + target = std::max(target, seconds); + }; + // Set values of FileInfo struct info.has_audio = true; info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1; @@ -775,34 +858,27 @@ void FFmpegReader::UpdateAudioInfo() { info.audio_timebase.den = aStream->time_base.den; // Get timebase of audio stream (if valid) and greater than the current duration - if (aStream->duration > 0 && aStream->duration > info.duration) { - // Get duration from audio stream - info.duration = aStream->duration * info.audio_timebase.ToDouble(); - } else if (pFormatCtx->duration > 0 && info.duration <= 0.0f) { - // Use the format's duration - info.duration = float(pFormatCtx->duration) / AV_TIME_BASE; + if (aStream->duration > 0) { + record_duration(audio_stream_duration_seconds, aStream->duration * info.audio_timebase.ToDouble()); + } + if (pFormatCtx->duration > 0) { + // Use the format's duration when stream duration is missing or shorter + record_duration(format_duration_seconds, static_cast(pFormatCtx->duration) / AV_TIME_BASE); } // Calculate duration from filesize and bitrate (if any) if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) { // Estimate from bitrate, total bytes, and framerate - info.duration = float(info.file_size) / info.video_bit_rate; - } - - // Check for an invalid video length - if (info.has_video && info.video_length <= 0) { - // Calculate the video length from the audio duration - info.video_length = info.duration * info.fps.ToDouble(); + record_duration(inferred_duration_seconds, static_cast(info.file_size) / info.video_bit_rate); } // Set video timebase (if no video stream was found) if (!info.has_video) { // Set a few important default video settings (so audio can be divided into frames) - info.fps.num = 24; + info.fps.num = 30; info.fps.den = 1; info.video_timebase.num = 1; - info.video_timebase.den = 24; - info.video_length = info.duration * info.fps.ToDouble(); + info.video_timebase.den = 30; info.width = 720; info.height = 480; @@ -817,10 +893,7 @@ void FFmpegReader::UpdateAudioInfo() { } } - // Fix invalid video lengths for certain types of files (MP3 for example) - if (info.has_video && ((info.duration * info.fps.ToDouble()) - info.video_length > 60)) { - info.video_length = info.duration * info.fps.ToDouble(); - } + ApplyDurationStrategy(); // Add audio metadata (if any found) AVDictionaryEntry *tag = NULL; @@ -837,6 +910,11 @@ void FFmpegReader::UpdateVideoInfo() { return; } + auto record_duration = [](double &target, double seconds) { + if (seconds > 0.0) + target = std::max(target, seconds); + }; + // Set values of FileInfo struct info.has_video = true; info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1; @@ -912,63 +990,35 @@ void FFmpegReader::UpdateVideoInfo() { info.video_timebase.den = pStream->time_base.den; // Set the duration in seconds, and video length (# of frames) - info.duration = pStream->duration * info.video_timebase.ToDouble(); + record_duration(video_stream_duration_seconds, pStream->duration * info.video_timebase.ToDouble()); // Check for valid duration (if found) - if (info.duration <= 0.0f && pFormatCtx->duration >= 0) { - // Use the format's duration - info.duration = float(pFormatCtx->duration) / AV_TIME_BASE; + if (pFormatCtx->duration >= 0) { + // Use the format's duration as another candidate + record_duration(format_duration_seconds, static_cast(pFormatCtx->duration) / AV_TIME_BASE); } // Calculate duration from filesize and bitrate (if any) - if (info.duration <= 0.0f && info.video_bit_rate > 0 && info.file_size > 0) { + if (info.video_bit_rate > 0 && info.file_size > 0) { // Estimate from bitrate, total bytes, and framerate - info.duration = float(info.file_size) / info.video_bit_rate; + record_duration(inferred_duration_seconds, static_cast(info.file_size) / info.video_bit_rate); } // Certain "image" formats do not have a valid duration - if (info.duration <= 0.0f && pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) { + if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 && + pStream->duration == AV_NOPTS_VALUE && pFormatCtx->duration == AV_NOPTS_VALUE) { // Force an "image" duration - info.duration = 60 * 60 * 1; // 1 hour duration - info.video_length = 1; + record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration info.has_single_image = true; } - - // Get the # of video frames (if found in stream) - // Only set this 1 time (this method can be called multiple times) - if (pStream->nb_frames > 0 && info.video_length <= 0) - { - info.video_length = pStream->nb_frames; - - // If the file format is animated GIF, override the video_length to be (duration * fps) rounded. - if (pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) - { - if (pStream->nb_frames > 1) { - // Animated gif (nb_frames does not take into delays and gaps) - info.video_length = round(info.duration * info.fps.ToDouble()); - } else { - // Static non-animated gif (set a default duration) - info.duration = 10.0; - } - } + // Static GIFs can have no usable duration; fall back to a small default + if (video_stream_duration_seconds <= 0.0 && format_duration_seconds <= 0.0 && + pFormatCtx && pFormatCtx->iformat && strcmp(pFormatCtx->iformat->name, "gif") == 0) { + record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration + info.has_single_image = true; } - // No duration found in stream of file - if (info.duration <= 0.0f) { - // No duration is found in the video stream - info.duration = -1; - info.video_length = -1; - is_duration_known = false; - } else { - // Yes, a duration was found - is_duration_known = true; - - // Calculate number of frames (if not already found in metadata) - // Only set this 1 time (this method can be called multiple times) - if (info.video_length <= 0) { - info.video_length = round(info.duration * info.fps.ToDouble()); - } - } + ApplyDurationStrategy(); // Add video metadata (if any) AVDictionaryEntry *tag = NULL; @@ -1056,7 +1106,7 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { int packet_error = -1; // Debug output - ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "max_concurrent_frames", max_concurrent_frames); + ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame); // Loop through the stream until the correct frame is found while (true) { @@ -1545,6 +1595,9 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { max_width = info.width * max_scale_x * preview_ratio; max_height = info.height * max_scale_y * preview_ratio; } + + // If a crop effect is resizing the image, request enough pixels to preserve detail + ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height); } // Determine if image needs to be scaled (for performance reasons) @@ -1894,7 +1947,7 @@ void FFmpegReader::Seek(int64_t requested_frame) { seek_count++; // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking) - int buffer_amount = std::max(max_concurrent_frames, 8); + int buffer_amount = 12; if (requested_frame - buffer_amount < 20) { // prevent Open() from seeking again is_seeking = true; @@ -2463,6 +2516,18 @@ Json::Value FFmpegReader::JsonValue() const { Json::Value root = ReaderBase::JsonValue(); // get parent properties root["type"] = "FFmpegReader"; root["path"] = path; + switch (duration_strategy) { + case DurationStrategy::VideoPreferred: + root["duration_strategy"] = "VideoPreferred"; + break; + case DurationStrategy::AudioPreferred: + root["duration_strategy"] = "AudioPreferred"; + break; + case DurationStrategy::LongestStream: + default: + root["duration_strategy"] = "LongestStream"; + break; + } // return JsonValue return root; @@ -2492,6 +2557,16 @@ void FFmpegReader::SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) if (!root["path"].isNull()) path = root["path"].asString(); + if (!root["duration_strategy"].isNull()) { + const std::string strategy = root["duration_strategy"].asString(); + if (strategy == "VideoPreferred") { + duration_strategy = DurationStrategy::VideoPreferred; + } else if (strategy == "AudioPreferred") { + duration_strategy = DurationStrategy::AudioPreferred; + } else { + duration_strategy = DurationStrategy::LongestStream; + } + } // Re-Open path, and re-init everything (if needed) if (is_open) { diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 295cdcf3a..f264754c3 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -17,6 +17,7 @@ #define OPENSHOT_FFMPEG_READER_H #include "ReaderBase.h" +#include "Enums.h" // Include FFmpeg headers and macros #include "FFmpegUtilities.h" @@ -116,7 +117,7 @@ namespace openshot { bool is_duration_known; bool check_interlace; bool check_fps; - int max_concurrent_frames; + DurationStrategy duration_strategy; CacheMemory working_cache; AudioLocation previous_packet_location; @@ -149,6 +150,12 @@ namespace openshot { int64_t NO_PTS_OFFSET; PacketStatus packet_status; + // Duration bookkeeping + double video_stream_duration_seconds = 0.0; + double audio_stream_duration_seconds = 0.0; + double format_duration_seconds = 0.0; + double inferred_duration_seconds = 0.0; + // Cached conversion contexts and frames for performance SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context SWRCONTEXT *avr_ctx = nullptr; ///< Cached audio resample context @@ -197,6 +204,12 @@ namespace openshot { /// Check if there's an album art bool HasAlbumArt(); + /// Decide which duration to use based on the configured strategy + double PickDurationSeconds() const; + + /// Apply the chosen duration to info.duration and info.video_length + void ApplyDurationStrategy(); + /// Remove partial frames due to seek bool IsPartialFrame(int64_t requested_frame); @@ -244,6 +257,12 @@ namespace openshot { /// @param path The filesystem location to load /// @param inspect_reader if true (the default), automatically open the media file and loads frame 1. FFmpegReader(const std::string& path, bool inspect_reader=true); + /// @brief Constructor for FFmpegReader with duration strategy. + /// + /// @param path The filesystem location to load + /// @param duration_strategy Which duration source to prioritize + /// @param inspect_reader if true (the default), automatically open the media file and loads frame 1. + FFmpegReader(const std::string& path, DurationStrategy duration_strategy, bool inspect_reader=true); /// Destructor virtual ~FFmpegReader(); diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 79cbc21c6..b1feafe3f 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -126,17 +126,18 @@ void FFmpegWriter::auto_detect_format() { // Determine what format to use when encoding this output filename oc->oformat = av_guess_format(NULL, path.c_str(), NULL); if (oc->oformat == nullptr) { - throw InvalidFormat( - "Could not deduce output format from file extension.", path); + throw InvalidFormat("Could not deduce output format from file extension.", path); } - // Update video codec name - if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video) - info.vcodec = avcodec_find_encoder(oc->oformat->video_codec)->name; - - // Update audio codec name - if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio) - info.acodec = avcodec_find_encoder(oc->oformat->audio_codec)->name; + // Update video & audio codec name + if (oc->oformat->video_codec != AV_CODEC_ID_NONE && info.has_video) { + const AVCodec *vcodec = avcodec_find_encoder(oc->oformat->video_codec); + info.vcodec = vcodec ? vcodec->name : std::string(); + } + if (oc->oformat->audio_codec != AV_CODEC_ID_NONE && info.has_audio) { + const AVCodec *acodec = avcodec_find_encoder(oc->oformat->audio_codec); + info.acodec = acodec ? acodec->name : std::string(); + } } // initialize streams diff --git a/src/Frame.cpp b/src/Frame.cpp index f799bcea9..2e785fb2a 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -48,7 +48,7 @@ Frame::Frame(int64_t number, int width, int height, std::string color, int sampl channels(channels), channel_layout(LAYOUT_STEREO), sample_rate(44100), has_audio_data(false), has_image_data(false), - max_audio_sample(0) + max_audio_sample(0), audio_is_increasing(true) { // zero (fill with silence) the audio buffer audio->clear(); @@ -96,6 +96,7 @@ void Frame::DeepCopy(const Frame& other) pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den); color = other.color; max_audio_sample = other.max_audio_sample; + audio_is_increasing = other.audio_is_increasing; if (other.image) image = std::make_shared(*(other.image)); @@ -801,14 +802,16 @@ void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout max_audio_sample = length; } -// Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times -// you invoke this method) -void Frame::ReverseAudio() { - if (audio && !audio_reversed) { +/// Set the direction of the audio buffer of this frame +void Frame::SetAudioDirection(bool is_increasing) { + if (audio && !audio_is_increasing && is_increasing) { + // Forward audio buffer + audio->reverse(0, audio->getNumSamples()); + } else if (audio && audio_is_increasing && !is_increasing) { // Reverse audio buffer audio->reverse(0, audio->getNumSamples()); - audio_reversed = true; } + audio_is_increasing = is_increasing; } // Add audio samples to a specific channel @@ -838,8 +841,8 @@ void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, if (new_length > max_audio_sample) max_audio_sample = new_length; - // Reset audio reverse flag - audio_reversed = false; + // Reset audio direction + audio_is_increasing = true; } // Apply gain ramp (i.e. fading volume) @@ -995,6 +998,6 @@ void Frame::AddAudioSilence(int numSamples) // Calculate max audio sample added max_audio_sample = numSamples; - // Reset audio reverse flag - audio_reversed = false; + // Reset audio direction + audio_is_increasing = true; } diff --git a/src/Frame.h b/src/Frame.h index 528a69b85..7e8f26c55 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -102,7 +102,7 @@ namespace openshot int sample_rate; std::string color; int64_t max_audio_sample; ///< The max audio sample count added to this frame - bool audio_reversed; ///< Keep track of audio reversal (i.e. time keyframe) + bool audio_is_increasing; ///< Keep track of audio direction (i.e. related to time keyframe) #ifdef USE_OPENCV cv::Mat imagecv; ///< OpenCV image. It will always be in BGR format @@ -244,9 +244,8 @@ namespace openshot /// Set the original sample rate of this frame's audio data void SampleRate(int orig_sample_rate) { sample_rate = orig_sample_rate; }; - /// Reverse the audio buffer of this frame (will only reverse a single time, regardless of how many times - /// you invoke this method) - void ReverseAudio(); + /// Set the direction of the audio buffer of this frame + void SetAudioDirection(bool is_increasing); /// Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM void Save(std::string path, float scale, std::string format="PNG", int quality=100); diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index 8c05eb56e..531cd571c 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -17,6 +17,7 @@ #include "FrameMapper.h" #include "Exceptions.h" #include "Clip.h" +#include "MemoryTrim.h" #include "ZmqLogger.h" using namespace std; @@ -440,6 +441,34 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) // Find parent properties (if any) Clip *parent = static_cast(ParentClip()); bool is_increasing = true; + bool direction_flipped = false; + + { + const std::lock_guard lock(directionMutex); + + // One-shot: if a hint exists, consume it for THIS call, regardless of frame number. + if (have_hint) { + is_increasing = hint_increasing; + have_hint = false; + } else if (previous_frame > 0 && std::llabs(requested_frame - previous_frame) == 1) { + // Infer from request order when adjacent + is_increasing = (requested_frame > previous_frame); + } else if (last_dir_initialized) { + // Reuse last known direction if non-adjacent and no hint + is_increasing = last_is_increasing; + } else { + is_increasing = true; // default on first call + } + + // Detect flips so we can reset SR context + if (!last_dir_initialized) { + last_is_increasing = is_increasing; + last_dir_initialized = true; + } else if (last_is_increasing != is_increasing) { + direction_flipped = true; + last_is_increasing = is_increasing; + } + } if (parent) { float position = parent->Position(); float start = parent->Start(); @@ -448,10 +477,6 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) // since this heavily affects frame #s and audio mappings is_dirty = true; } - - // Determine direction of parent clip at this frame (forward or reverse direction) - // This is important for reversing audio in our resampler, for smooth reversed audio. - is_increasing = parent->time.IsIncreasing(requested_frame); } // Check if mappings are dirty (and need to be recalculated) @@ -548,13 +573,13 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) if (need_resampling) { - // Check for non-adjacent frame requests - so the resampler can be reset - if (abs(frame->number - previous_frame) > 1) { + // Reset resampler when non-adjacent request OR playback direction flips + if (direction_flipped || (previous_frame > 0 && std::llabs(requested_frame - previous_frame) > 1)) { if (avr) { // Delete resampler (if exists) SWR_CLOSE(avr); SWR_FREE(&avr); - avr = NULL; + avr = nullptr; } } @@ -630,9 +655,8 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) starting_frame++; } - // Reverse audio (if needed) - if (!is_increasing) - frame->ReverseAudio(); + // Set audio direction + frame->SetAudioDirection(is_increasing); // Resample audio on frame (if needed) if (need_resampling) @@ -722,6 +746,9 @@ void FrameMapper::Close() SWR_FREE(&avr); avr = NULL; } + + // Release free’d arenas back to OS after heavy teardown + TrimMemoryToOS(true); } @@ -818,7 +845,7 @@ void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldow final_cache.Clear(); // Adjust cache size based on size of frame and audio - final_cache.SetMaxBytesFromInfo(OPEN_MP_NUM_PROCESSORS, info.width, info.height, info.sample_rate, info.channels); + final_cache.SetMaxBytesFromInfo(24, info.width, info.height, info.sample_rate, info.channels); // Deallocate resample buffer if (avr) { @@ -1039,3 +1066,11 @@ int64_t FrameMapper::AdjustFrameNumber(int64_t clip_frame_number) { return frame_number; } + +// Set direction hint for the next call to GetFrame +void FrameMapper::SetDirectionHint(const bool increasing) +{ + const std::lock_guard lock(directionMutex); + hint_increasing = increasing; + have_hint = true; +} diff --git a/src/FrameMapper.h b/src/FrameMapper.h index 55ad74749..a932c2d4f 100644 --- a/src/FrameMapper.h +++ b/src/FrameMapper.h @@ -204,6 +204,13 @@ namespace openshot int64_t previous_frame; // Used during resampling, to determine when a large gap is detected SWRCONTEXT *avr; // Audio resampling context object + // Time curve / direction + std::recursive_mutex directionMutex; + bool have_hint = false; + bool hint_increasing = true; + bool last_is_increasing = true; + bool last_dir_initialized = false; + // Audio resampler (if resampling audio) openshot::AudioResampler *resampler; @@ -273,6 +280,9 @@ namespace openshot /// Open the internal reader void Open() override; + /// Set time-curve informed direction hint (from Clip class) for the next call to GetFrame + void SetDirectionHint(const bool increasing); + /// Print all of the original frames and which new frames they map to void PrintMapping(std::ostream* out=&std::cout); diff --git a/src/IdGenerator.h b/src/IdGenerator.h new file mode 100644 index 000000000..85a10dd51 --- /dev/null +++ b/src/IdGenerator.h @@ -0,0 +1,36 @@ +/* + * @file + * @brief Header file for generating random identifier strings + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_ID_GENERATOR_H +#define OPENSHOT_ID_GENERATOR_H + +#include +#include + +namespace openshot { + + class IdGenerator { + public: + static inline std::string Generate(int length = 8) { + static const char charset[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dist(0, static_cast(sizeof(charset) - 2)); + + std::string result; + result.reserve(length); + for (int i = 0; i < length; ++i) + result += charset[dist(gen)]; + return result; +} +}; + +} // namespace openshot + +#endif // OPENSHOT_ID_GENERATOR_H diff --git a/src/ImageReader.cpp b/src/ImageReader.cpp index 1240a7f67..e3342a414 100644 --- a/src/ImageReader.cpp +++ b/src/ImageReader.cpp @@ -61,10 +61,11 @@ void ImageReader::Open() info.width = image->size().width(); info.height = image->size().height(); info.pixel_ratio = openshot::Fraction(1, 1); - info.duration = 60 * 60 * 1; // 1 hour duration info.fps = openshot::Fraction(30, 1); info.video_timebase = info.fps.Reciprocal(); - info.video_length = std::round(info.duration * info.fps.ToDouble()); + // Default still-image duration: 1 hour, aligned to fps + info.video_length = 60 * 60 * info.fps.num; // 3600 seconds * 30 fps + info.duration = static_cast(info.video_length / info.fps.ToDouble()); // Calculate the DAR (display aspect ratio) Fraction dar( diff --git a/src/KeyFrame.cpp b/src/KeyFrame.cpp index 1df3f3c06..8c7615816 100644 --- a/src/KeyFrame.cpp +++ b/src/KeyFrame.cpp @@ -399,7 +399,7 @@ void Keyframe::SetJsonValue(const Json::Value root) { double Keyframe::GetDelta(int64_t index) const { if (index < 1) return 0.0; if (index == 1 && !Points.empty()) return Points[0].co.Y; - if (index >= GetLength()) return 0.0; + if (index > GetLength()) return 1.0; return GetValue(index) - GetValue(index - 1); } diff --git a/src/MemoryTrim.cpp b/src/MemoryTrim.cpp new file mode 100644 index 000000000..8a73ec501 --- /dev/null +++ b/src/MemoryTrim.cpp @@ -0,0 +1,80 @@ +/** + * @file + * @brief Cross-platform helper to encourage returning freed memory to the OS + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "MemoryTrim.h" + +#include +#include +#include + +#if defined(__GLIBC__) +#include +#elif defined(_WIN32) +#include +#elif defined(__APPLE__) +#include +#endif + +namespace { +// Limit trim attempts to once per interval to avoid spamming platform calls +constexpr uint64_t kMinTrimIntervalMs = 1000; // 1s debounce +std::atomic g_last_trim_ms{0}; +std::atomic g_trim_in_progress{false}; + +uint64_t NowMs() { + using namespace std::chrono; + return duration_cast(steady_clock::now().time_since_epoch()).count(); +} +} // namespace + +namespace openshot { + +bool TrimMemoryToOS(bool force) noexcept { + const uint64_t now_ms = NowMs(); + const uint64_t last_ms = g_last_trim_ms.load(std::memory_order_relaxed); + + // Skip if we recently trimmed (unless forced) + if (!force && now_ms - last_ms < kMinTrimIntervalMs) + return false; + + // Only one trim attempt runs at a time + bool expected = false; + if (!g_trim_in_progress.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + return false; + + bool did_trim = false; + +#if defined(__GLIBC__) + // GLIBC exposes malloc_trim to release free arenas back to the OS + malloc_trim(0); + did_trim = true; +#elif defined(_WIN32) + // MinGW/MSYS2 expose _heapmin to compact the CRT heap + _heapmin(); + did_trim = true; +#elif defined(__APPLE__) + // macOS uses the malloc zone API to relieve memory pressure + malloc_zone_t* zone = malloc_default_zone(); + malloc_zone_pressure_relief(zone, 0); + did_trim = true; +#else + // Platforms without a known trimming API + did_trim = false; +#endif + + if (did_trim) + g_last_trim_ms.store(now_ms, std::memory_order_relaxed); + + g_trim_in_progress.store(false, std::memory_order_release); + return did_trim; +} + +} // namespace openshot diff --git a/src/MemoryTrim.h b/src/MemoryTrim.h new file mode 100644 index 000000000..943fa0ae6 --- /dev/null +++ b/src/MemoryTrim.h @@ -0,0 +1,30 @@ +/** + * @file + * @brief Cross-platform helper to encourage returning freed memory to the OS + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +namespace openshot { + +/** + * @brief Attempt to return unused heap memory to the operating system. + * + * This maps to the appropriate platform-specific API where available. + * The call is safe to invoke on any supported platform; on unsupported + * platforms it will simply return false without doing anything. + * Calls are rate-limited internally (1s debounce) and single-flight. A forced + * call bypasses the debounce but still honors the single-flight guard. + * + * @param force If true, bypass the debounce interval (useful for teardown). + * @return true if a platform-specific trim call was made, false otherwise. + */ +bool TrimMemoryToOS(bool force = false) noexcept; + +} // namespace openshot diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index a7336b07d..643ed0a64 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -48,6 +48,14 @@ namespace openshot // Is cache ready for playback (pre-roll) bool VideoCacheThread::isReady() { + if (!reader) { + return false; + } + + if (min_frames_ahead < 0) { + return true; + } + return (cached_frame_count > min_frames_ahead); } @@ -96,11 +104,18 @@ namespace openshot if (start_preroll) { userSeeked = true; - if (!reader->GetCache()->Contains(new_position)) + CacheBase* cache = reader ? reader->GetCache() : nullptr; + + if (cache && !cache->Contains(new_position)) { // If user initiated seek, and current frame not found ( Timeline* timeline = static_cast(reader); timeline->ClearAllCache(); + cached_frame_count = 0; + } + else if (cache) + { + cached_frame_count = cache->Count(); } } requested_display_frame = new_position; @@ -131,6 +146,7 @@ namespace openshot // If paused and playhead not in cache, clear everything Timeline* timeline = static_cast(reader); timeline->ClearAllCache(); + cached_frame_count = 0; return true; } return false; @@ -184,7 +200,7 @@ namespace openshot try { auto framePtr = reader->GetFrame(next_frame); cache->Add(framePtr); - ++cached_frame_count; + cached_frame_count = cache->Count(); } catch (const OutOfBoundsFrame&) { break; @@ -211,8 +227,10 @@ namespace openshot Settings* settings = Settings::Instance(); CacheBase* cache = reader ? reader->GetCache() : nullptr; - // If caching disabled or no reader, sleep briefly + // If caching disabled or no reader, mark cache as ready and sleep briefly if (!settings->ENABLE_PLAYBACK_CACHING || !cache) { + cached_frame_count = (cache ? cache->Count() : 0); + min_frames_ahead = -1; std::this_thread::sleep_for(double_micro_sec(50000)); continue; } @@ -225,6 +243,8 @@ namespace openshot int64_t playhead = requested_display_frame; bool paused = (speed == 0); + cached_frame_count = cache->Count(); + // Compute effective direction (±1) int dir = computeDirection(); if (speed != 0) { @@ -282,6 +302,16 @@ namespace openshot } int64_t ahead_count = static_cast(capacity * settings->VIDEO_CACHE_PERCENT_AHEAD); + int64_t window_size = ahead_count + 1; + if (window_size < 1) { + window_size = 1; + } + int64_t ready_target = window_size - 1; + if (ready_target < 0) { + ready_target = 0; + } + int64_t configured_min = settings->VIDEO_CACHE_MIN_PREROLL_FRAMES; + min_frames_ahead = std::min(configured_min, ready_target); // If paused and playhead is no longer in cache, clear everything bool did_clear = clearCacheIfPaused(playhead, paused, cache); diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index 1fd6decd5..72f502e04 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -166,7 +166,7 @@ namespace openshot int64_t requested_display_frame; ///< Frame index the user requested. int64_t current_display_frame; ///< Currently displayed frame (unused here, reserved). - int64_t cached_frame_count; ///< Count of frames currently added to cache. + int64_t cached_frame_count; ///< Estimated count of frames currently stored in cache. int64_t min_frames_ahead; ///< Minimum number of frames considered “ready” (pre-roll). int64_t timeline_max_frame; ///< Highest valid frame index in the timeline. diff --git a/src/QtImageReader.cpp b/src/QtImageReader.cpp index 801c021f7..ffd20862c 100644 --- a/src/QtImageReader.cpp +++ b/src/QtImageReader.cpp @@ -16,7 +16,9 @@ #include "CacheMemory.h" #include "Exceptions.h" #include "Timeline.h" +#include "effects/CropHelpers.h" +#include #include #include #include @@ -100,12 +102,13 @@ void QtImageReader::Open() } info.pixel_ratio.num = 1; info.pixel_ratio.den = 1; - info.duration = 60 * 60 * 1; // 1 hour duration info.fps.num = 30; info.fps.den = 1; info.video_timebase.num = 1; info.video_timebase.den = 30; - info.video_length = round(info.duration * info.fps.ToDouble()); + // Default still-image duration: 1 hour, aligned to fps + info.video_length = 60 * 60 * info.fps.num; // 3600 seconds * 30 fps + info.duration = static_cast(info.video_length / info.fps.ToDouble()); // Calculate the DAR (display aspect ratio) Fraction size(info.width * info.pixel_ratio.num, info.height * info.pixel_ratio.den); @@ -243,6 +246,9 @@ QSize QtImageReader::calculate_max_size() { max_width = info.width * max_scale_x * preview_ratio; max_height = info.height * max_scale_y * preview_ratio; } + + // If a crop effect is resizing the image, request enough pixels to preserve detail + ApplyCropResizeScale(parent, info.width, info.height, max_width, max_height); } // Return new QSize of the current max size diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 9f8efb58b..26fa96bd1 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -21,13 +21,15 @@ #include #include +#include +#include +#include using namespace openshot; // Default Constructor for the timeline (which sets the canvas width and height) Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int channels, ChannelLayout channel_layout) : - is_open(false), auto_map_clips(true), managed_cache(true), path(""), - max_concurrent_frames(OPEN_MP_NUM_PROCESSORS), max_time(0.0) + is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -67,7 +69,7 @@ Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int cha // Init cache final_cache = new CacheMemory(); - final_cache->SetMaxBytesFromInfo(max_concurrent_frames * 4, info.width, info.height, info.sample_rate, info.channels); + final_cache->SetMaxBytesFromInfo(24, info.width, info.height, info.sample_rate, info.channels); } // Delegating constructor that copies parameters from a provided ReaderInfo @@ -77,8 +79,7 @@ Timeline::Timeline(const ReaderInfo info) : Timeline::Timeline( // Constructor for the timeline (which loads a JSON structure from a file path, and initializes a timeline) Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) : - is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), - max_concurrent_frames(OPEN_MP_NUM_PROCESSORS), max_time(0.0) { + is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -200,7 +201,7 @@ Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) // Init cache final_cache = new CacheMemory(); - final_cache->SetMaxBytesFromInfo(max_concurrent_frames * 4, info.width, info.height, info.sample_rate, info.channels); + final_cache->SetMaxBytesFromInfo(24, info.width, info.height, info.sample_rate, info.channels); } Timeline::~Timeline() { @@ -357,6 +358,9 @@ void Timeline::AddClip(Clip* clip) // Add an effect to the timeline void Timeline::AddEffect(EffectBase* effect) { + // Get lock (prevent getting frames while this happens) + const std::lock_guard guard(getFrameMutex); + // Assign timeline to effect effect->ParentTimeline(this); @@ -370,14 +374,16 @@ void Timeline::AddEffect(EffectBase* effect) // Remove an effect from the timeline void Timeline::RemoveEffect(EffectBase* effect) { + // Get lock (prevent getting frames while this happens) + const std::lock_guard guard(getFrameMutex); + effects.remove(effect); // Delete effect object (if timeline allocated it) - bool allocated = allocated_effects.count(effect); - if (allocated) { + if (allocated_effects.count(effect)) { + allocated_effects.erase(effect); // erase before nulling the pointer delete effect; effect = NULL; - allocated_effects.erase(effect); } // Sort effects @@ -393,11 +399,10 @@ void Timeline::RemoveClip(Clip* clip) clips.remove(clip); // Delete clip object (if timeline allocated it) - bool allocated = allocated_clips.count(clip); - if (allocated) { + if (allocated_clips.count(clip)) { + allocated_clips.erase(clip); // erase before nulling the pointer delete clip; clip = NULL; - allocated_clips.erase(clip); } // Sort clips @@ -467,9 +472,18 @@ double Timeline::GetMaxTime() { // Compute the highest frame# based on the latest time and FPS int64_t Timeline::GetMaxFrame() { - double fps = info.fps.ToDouble(); - auto max_time = GetMaxTime(); - return std::round(max_time * fps); + const double fps = info.fps.ToDouble(); + const double t = GetMaxTime(); + // Inclusive start, exclusive end -> ceil at the end boundary + return static_cast(std::ceil(t * fps)); +} + +// Compute the first frame# based on the first clip position +int64_t Timeline::GetMinFrame() { + const double fps = info.fps.ToDouble(); + const double t = GetMinTime(); + // Inclusive start -> floor at the start boundary, then 1-index + return static_cast(std::floor(t * fps)) + 1; } // Compute the start time of the first timeline clip @@ -478,13 +492,6 @@ double Timeline::GetMinTime() { return min_time; } -// Compute the first frame# based on the first clip position -int64_t Timeline::GetMinFrame() { - double fps = info.fps.ToDouble(); - auto min_time = GetMinTime(); - return std::round(min_time * fps) + 1; -} - // Apply a FrameMapper to a clip which matches the settings of this timeline void Timeline::apply_mapper_to_clip(Clip* clip) { @@ -549,8 +556,9 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int for (auto effect : effects) { // Does clip intersect the current requested time - long effect_start_position = round(effect->Position() * info.fps.ToDouble()) + 1; - long effect_end_position = round((effect->Position() + (effect->Duration())) * info.fps.ToDouble()); + const double fpsD = info.fps.ToDouble(); + int64_t effect_start_position = static_cast(std::llround(effect->Position() * fpsD)) + 1; + int64_t effect_end_position = static_cast(std::llround((effect->Position() + effect->Duration()) * fpsD)); bool does_effect_intersect = (effect_start_position <= timeline_frame_number && effect_end_position >= timeline_frame_number && effect->Layer() == layer); @@ -558,8 +566,8 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int if (does_effect_intersect) { // Determine the frame needed for this clip (based on the position on the timeline) - long effect_start_frame = (effect->Start() * info.fps.ToDouble()) + 1; - long effect_frame_number = timeline_frame_number - effect_start_position + effect_start_frame; + int64_t effect_start_frame = static_cast(std::llround(effect->Start() * fpsD)) + 1; + int64_t effect_frame_number = timeline_frame_number - effect_start_position + effect_start_frame; if (!options->is_top_clip) continue; // skip effect, if overlapped/covered by another clip on same layer @@ -624,14 +632,13 @@ std::shared_ptr Timeline::GetOrCreateFrame(std::shared_ptr backgro void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume) { // Create timeline options (with details about this current frame request) - TimelineInfoStruct* options = new TimelineInfoStruct(); - options->is_top_clip = is_top_clip; - options->is_before_clip_keyframes = true; + TimelineInfoStruct options{}; + options.is_top_clip = is_top_clip; + options.is_before_clip_keyframes = true; // Get the clip's frame, composited on top of the current timeline frame std::shared_ptr source_frame; - source_frame = GetOrCreateFrame(new_frame, source_clip, clip_frame_number, options); - delete options; + source_frame = GetOrCreateFrame(new_frame, source_clip, clip_frame_number, &options); // No frame found... so bail if (!source_frame) @@ -654,6 +661,12 @@ void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, in "clip_frame_number", clip_frame_number); if (source_frame->GetAudioChannelsCount() == info.channels && source_clip->has_audio.GetInt(clip_frame_number) != 0) + { + // Ensure timeline frame matches the source samples once per frame + if (new_frame->GetAudioSamplesCount() != source_frame->GetAudioSamplesCount()){ + new_frame->ResizeAudio(info.channels, source_frame->GetAudioSamplesCount(), info.sample_rate, info.channel_layout); + } + for (int channel = 0; channel < source_frame->GetAudioChannelsCount(); channel++) { // Get volume from previous frame and this frame @@ -690,18 +703,11 @@ void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, in if (!isEqual(previous_volume, 1.0) || !isEqual(volume, 1.0)) source_frame->ApplyGainRamp(channel_mapping, 0, source_frame->GetAudioSamplesCount(), previous_volume, volume); - // TODO: Improve FrameMapper (or Timeline) to always get the correct number of samples per frame. - // Currently, the ResampleContext sometimes leaves behind a few samples for the next call, and the - // number of samples returned is variable... and does not match the number expected. - // This is a crude solution at best. =) - if (new_frame->GetAudioSamplesCount() != source_frame->GetAudioSamplesCount()){ - // Force timeline frame to match the source frame - new_frame->ResizeAudio(info.channels, source_frame->GetAudioSamplesCount(), info.sample_rate, info.channel_layout); - } // Copy audio samples (and set initial volume). Mix samples with existing audio samples. The gains are added together, to // be sure to set the gain's correctly, so the sum does not exceed 1.0 (of audio distortion will happen). new_frame->AddAudio(false, channel_mapping, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), 1.0); } + } else // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1003,67 +1009,90 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) "clips.size()", clips.size(), "nearby_clips.size()", nearby_clips.size()); - // Find Clips near this time + // Precompute per-clip timing for this requested frame + struct ClipInfo { + Clip* clip; + int64_t start_pos; + int64_t end_pos; + int64_t start_frame; + int64_t frame_number; + bool intersects; + }; + std::vector clip_infos; + clip_infos.reserve(nearby_clips.size()); + const double fpsD = info.fps.ToDouble(); + for (auto clip : nearby_clips) { - long clip_start_position = round(clip->Position() * info.fps.ToDouble()) + 1; - long clip_end_position = round((clip->Position() + clip->Duration()) * info.fps.ToDouble()); - bool does_clip_intersect = (clip_start_position <= requested_frame && clip_end_position >= requested_frame); + int64_t start_pos = static_cast(std::llround(clip->Position() * fpsD)) + 1; + int64_t end_pos = static_cast(std::llround((clip->Position() + clip->Duration()) * fpsD)); + bool intersects = (start_pos <= requested_frame && end_pos >= requested_frame); + int64_t start_frame = static_cast(std::llround(clip->Start() * fpsD)) + 1; + int64_t frame_number = requested_frame - start_pos + start_frame; + clip_infos.push_back({clip, start_pos, end_pos, start_frame, frame_number, intersects}); + } + // Determine top clip per layer (linear, no nested loop) + std::unordered_map top_start_for_layer; + std::unordered_map top_clip_for_layer; + for (const auto& ci : clip_infos) { + if (!ci.intersects) continue; + const int layer = ci.clip->Layer(); + auto it = top_start_for_layer.find(layer); + if (it == top_start_for_layer.end() || ci.start_pos > it->second) { + top_start_for_layer[layer] = ci.start_pos; // strictly greater to match prior logic + top_clip_for_layer[layer] = ci.clip; + } + } + + // Compute max_volume across all overlapping clips once + float max_volume_sum = 0.0f; + for (const auto& ci : clip_infos) { + if (!ci.intersects) continue; + if (ci.clip->Reader() && ci.clip->Reader()->info.has_audio && + ci.clip->has_audio.GetInt(ci.frame_number) != 0) { + max_volume_sum += static_cast(ci.clip->volume.GetValue(ci.frame_number)); + } + } + + // Compose intersecting clips in a single pass + for (const auto& ci : clip_infos) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( "Timeline::GetFrame (Does clip intersect)", "requested_frame", requested_frame, - "clip->Position()", clip->Position(), - "clip->Duration()", clip->Duration(), - "does_clip_intersect", does_clip_intersect); + "clip->Position()", ci.clip->Position(), + "clip->Duration()", ci.clip->Duration(), + "does_clip_intersect", ci.intersects); // Clip is visible - if (does_clip_intersect) { - // Determine if clip is "top" clip on this layer (only happens when multiple clips are overlapping) - bool is_top_clip = true; - float max_volume = 0.0; - for (auto nearby_clip : nearby_clips) { - long nearby_clip_start_position = round(nearby_clip->Position() * info.fps.ToDouble()) + 1; - long nearby_clip_end_position = round((nearby_clip->Position() + nearby_clip->Duration()) * info.fps.ToDouble()) + 1; - long nearby_clip_start_frame = (nearby_clip->Start() * info.fps.ToDouble()) + 1; - long nearby_clip_frame_number = requested_frame - nearby_clip_start_position + nearby_clip_start_frame; - - // Determine if top clip - if (clip->Id() != nearby_clip->Id() && clip->Layer() == nearby_clip->Layer() && - nearby_clip_start_position <= requested_frame && nearby_clip_end_position >= requested_frame && - nearby_clip_start_position > clip_start_position && is_top_clip == true) { - is_top_clip = false; - } - - // Determine max volume of overlapping clips - if (nearby_clip->Reader() && nearby_clip->Reader()->info.has_audio && - nearby_clip->has_audio.GetInt(nearby_clip_frame_number) != 0 && - nearby_clip_start_position <= requested_frame && nearby_clip_end_position >= requested_frame) { - max_volume += nearby_clip->volume.GetValue(nearby_clip_frame_number); - } - } + if (ci.intersects) { + // Is this the top clip on its layer? + bool is_top_clip = false; + const int layer = ci.clip->Layer(); + auto top_it = top_clip_for_layer.find(layer); + if (top_it != top_clip_for_layer.end()) + is_top_clip = (top_it->second == ci.clip); // Determine the frame needed for this clip (based on the position on the timeline) - long clip_start_frame = (clip->Start() * info.fps.ToDouble()) + 1; - long clip_frame_number = requested_frame - clip_start_position + clip_start_frame; + int64_t clip_frame_number = ci.frame_number; // Debug output ZmqLogger::Instance()->AppendDebugMethod( "Timeline::GetFrame (Calculate clip's frame #)", - "clip->Position()", clip->Position(), - "clip->Start()", clip->Start(), + "clip->Position()", ci.clip->Position(), + "clip->Start()", ci.clip->Start(), "info.fps.ToFloat()", info.fps.ToFloat(), "clip_frame_number", clip_frame_number); // Add clip's frame as layer - add_layer(new_frame, clip, clip_frame_number, is_top_clip, max_volume); + add_layer(new_frame, ci.clip, clip_frame_number, is_top_clip, max_volume_sum); } else { // Debug output ZmqLogger::Instance()->AppendDebugMethod( "Timeline::GetFrame (clip does not intersect)", "requested_frame", requested_frame, - "does_clip_intersect", does_clip_intersect); + "does_clip_intersect", ci.intersects); } } // end clip loop @@ -1095,15 +1124,17 @@ std::vector Timeline::find_intersecting_clips(int64_t requested_frame, in std::vector matching_clips; // Calculate time of frame - float min_requested_frame = requested_frame; - float max_requested_frame = requested_frame + (number_of_frames - 1); + const int64_t min_requested_frame = requested_frame; + const int64_t max_requested_frame = requested_frame + (number_of_frames - 1); // Find Clips at this time + matching_clips.reserve(clips.size()); + const double fpsD = info.fps.ToDouble(); for (auto clip : clips) { // Does clip intersect the current requested time - long clip_start_position = round(clip->Position() * info.fps.ToDouble()) + 1; - long clip_end_position = round((clip->Position() + clip->Duration()) * info.fps.ToDouble()) + 1; + int64_t clip_start_position = static_cast(std::llround(clip->Position() * fpsD)) + 1; + int64_t clip_end_position = static_cast(std::llround((clip->Position() + clip->Duration()) * fpsD)) + 1; bool does_clip_intersect = (clip_start_position <= min_requested_frame || clip_start_position <= max_requested_frame) && @@ -1431,17 +1462,26 @@ void Timeline::apply_json_to_clips(Json::Value change) { // Update existing clip if (existing_clip) { + // Calculate start and end frames prior to the update + int64_t old_starting_frame = (existing_clip->Position() * info.fps.ToDouble()) + 1; + int64_t old_ending_frame = ((existing_clip->Position() + existing_clip->Duration()) * info.fps.ToDouble()) + 1; + // Update clip properties from JSON existing_clip->SetJsonValue(change["value"]); - // Calculate start and end frames that this impacts, and remove those frames from the cache - int64_t old_starting_frame = (existing_clip->Position() * info.fps.ToDouble()) + 1; - int64_t old_ending_frame = ((existing_clip->Position() + existing_clip->Duration()) * info.fps.ToDouble()) + 1; + // Calculate new start and end frames after the update + int64_t new_starting_frame = (existing_clip->Position() * info.fps.ToDouble()) + 1; + int64_t new_ending_frame = ((existing_clip->Position() + existing_clip->Duration()) * info.fps.ToDouble()) + 1; + + // Remove both the old and new ranges from the timeline cache final_cache->Remove(old_starting_frame - 8, old_ending_frame + 8); + final_cache->Remove(new_starting_frame - 8, new_ending_frame + 8); // Remove cache on clip's Reader (if found) - if (existing_clip->Reader() && existing_clip->Reader()->GetCache()) + if (existing_clip->Reader() && existing_clip->Reader()->GetCache()) { existing_clip->Reader()->GetCache()->Remove(old_starting_frame - 8, old_ending_frame + 8); + existing_clip->Reader()->GetCache()->Remove(new_starting_frame - 8, new_ending_frame + 8); + } // Apply framemapper (or update existing framemapper) if (auto_map_clips) { @@ -1464,13 +1504,6 @@ void Timeline::apply_json_to_clips(Json::Value change) { } - // Calculate start and end frames that this impacts, and remove those frames from the cache - if (!change["value"].isArray() && !change["value"]["position"].isNull()) { - int64_t new_starting_frame = (change["value"]["position"].asDouble() * info.fps.ToDouble()) + 1; - int64_t new_ending_frame = ((change["value"]["position"].asDouble() + change["value"]["end"].asDouble() - change["value"]["start"].asDouble()) * info.fps.ToDouble()) + 1; - final_cache->Remove(new_starting_frame - 8, new_ending_frame + 8); - } - // Re-Sort Clips (since they likely changed) sort_clips(); } @@ -1720,18 +1753,24 @@ void Timeline::ClearAllCache(bool deep) { // Loop through all clips try { for (const auto clip : clips) { - // Clear cache on clip - clip->Reader()->GetCache()->Clear(); - - // Clear nested Reader (if deep clear requested) - if (deep && clip->Reader()->Name() == "FrameMapper") { - FrameMapper *nested_reader = static_cast(clip->Reader()); - if (nested_reader->Reader() && nested_reader->Reader()->GetCache()) - nested_reader->Reader()->GetCache()->Clear(); + // Clear cache on clip and reader if present + if (clip->Reader()) { + if (auto rc = clip->Reader()->GetCache()) + rc->Clear(); + + // Clear nested Reader (if deep clear requested) + if (deep && clip->Reader()->Name() == "FrameMapper") { + FrameMapper *nested_reader = static_cast(clip->Reader()); + if (nested_reader->Reader()) { + if (auto nc = nested_reader->Reader()->GetCache()) + nc->Clear(); + } + } } // Clear clip cache - clip->GetCache()->Clear(); + if (auto cc = clip->GetCache()) + cc->Clear(); } } catch (const ReaderClosed & e) { // ... diff --git a/src/Timeline.h b/src/Timeline.h index e93d2a7f6..46cdfb315 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -46,12 +46,18 @@ namespace openshot { /// Comparison method for sorting clip pointers (by Layer and then Position). Clips are sorted /// from lowest layer to top layer (since that is the sequence they need to be combined), and then /// by position (left to right). - struct CompareClips{ - bool operator()( openshot::Clip* lhs, openshot::Clip* rhs){ - if( lhs->Layer() < rhs->Layer() ) return true; - if( lhs->Layer() == rhs->Layer() && lhs->Position() <= rhs->Position() ) return true; - return false; - }}; + struct CompareClips { + bool operator()(openshot::Clip* lhs, openshot::Clip* rhs) const { + // Strict-weak ordering (no <=) to keep sort well-defined + if (lhs == rhs) return false; // irreflexive + if (lhs->Layer() != rhs->Layer()) + return lhs->Layer() < rhs->Layer(); + if (lhs->Position() != rhs->Position()) + return lhs->Position() < rhs->Position(); + // Stable tie-breaker on address to avoid equivalence when layer/position match + return std::less()(lhs, rhs); + } + }; /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted /// from lowest layer to top layer (since that is sequence clips are combined), and then by @@ -159,7 +165,6 @@ namespace openshot { std::set allocated_frame_mappers; ///< all the frame mappers we allocated and must free bool managed_cache; ///< Does this timeline instance manage the cache object std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file - int max_concurrent_frames; ///< Max concurrent frames to process at one time double max_time; ///> The max duration (in seconds) of the timeline, based on the furthest clip (right edge) double min_time; ///> The min duration (in seconds) of the timeline, based on the position of the first clip (left edge) diff --git a/src/effects/AnalogTape.cpp b/src/effects/AnalogTape.cpp new file mode 100644 index 000000000..5ccedf867 --- /dev/null +++ b/src/effects/AnalogTape.cpp @@ -0,0 +1,453 @@ +/** + * @file + * @brief Source file for AnalogTape effect class + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "AnalogTape.h" +#include "Clip.h" +#include "Exceptions.h" +#include "ReaderBase.h" +#include "Timeline.h" + +#include +#include + +using namespace openshot; + +AnalogTape::AnalogTape() + : tracking(0.55), bleed(0.65), softness(0.40), noise(0.50), stripe(0.25f), + staticBands(0.20f), seed_offset(0) { + init_effect_details(); +} + +AnalogTape::AnalogTape(Keyframe t, Keyframe b, Keyframe s, Keyframe n, + Keyframe st, Keyframe sb, int seed) + : tracking(t), bleed(b), softness(s), noise(n), stripe(st), + staticBands(sb), seed_offset(seed) { + init_effect_details(); +} + +void AnalogTape::init_effect_details() { + InitEffectInfo(); + info.class_name = "AnalogTape"; + info.name = "Analog Tape"; + info.description = "Vintage home video wobble, bleed, and grain."; + info.has_video = true; + info.has_audio = false; +} + +static inline float lerp(float a, float b, float t) { return a + (b - a) * t; } + +std::shared_ptr AnalogTape::GetFrame(std::shared_ptr frame, + int64_t frame_number) { + std::shared_ptr img = frame->GetImage(); + int w = img->width(); + int h = img->height(); + int Uw = (w + 1) / 2; + int stride = img->bytesPerLine() / 4; + uint32_t *base = reinterpret_cast(img->bits()); + + if (w != last_w || h != last_h) { + last_w = w; + last_h = h; + Y.resize(w * h); + U.resize(Uw * h); + V.resize(Uw * h); + tmpY.resize(w * h); + tmpU.resize(Uw * h); + tmpV.resize(Uw * h); + dx.resize(h); + } + + +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) { + uint32_t *row = base + y * stride; + float *yrow = &Y[y * w]; + float *urow = &U[y * Uw]; + float *vrow = &V[y * Uw]; + for (int x2 = 0; x2 < Uw; ++x2) { + int x0 = x2 * 2; + uint32_t p0 = row[x0]; + float r0 = ((p0 >> 16) & 0xFF) / 255.0f; + float g0 = ((p0 >> 8) & 0xFF) / 255.0f; + float b0 = (p0 & 0xFF) / 255.0f; + float y0 = 0.299f * r0 + 0.587f * g0 + 0.114f * b0; + float u0 = -0.14713f * r0 - 0.28886f * g0 + 0.436f * b0; + float v0 = 0.615f * r0 - 0.51499f * g0 - 0.10001f * b0; + yrow[x0] = y0; + + float u, v; + if (x0 + 1 < w) { + uint32_t p1 = row[x0 + 1]; + float r1 = ((p1 >> 16) & 0xFF) / 255.0f; + float g1 = ((p1 >> 8) & 0xFF) / 255.0f; + float b1 = (p1 & 0xFF) / 255.0f; + float y1 = 0.299f * r1 + 0.587f * g1 + 0.114f * b1; + float u1 = -0.14713f * r1 - 0.28886f * g1 + 0.436f * b1; + float v1 = 0.615f * r1 - 0.51499f * g1 - 0.10001f * b1; + yrow[x0 + 1] = y1; + u = (u0 + u1) * 0.5f; + v = (v0 + v1) * 0.5f; + } else { + u = u0; + v = v0; + } + urow[x2] = u; + vrow[x2] = v; + } + } + + Fraction fps(1, 1); + Clip *clip = (Clip *)ParentClip(); + Timeline *timeline = nullptr; + if (clip && clip->ParentTimeline()) + timeline = (Timeline *)clip->ParentTimeline(); + else if (ParentTimeline()) + timeline = (Timeline *)ParentTimeline(); + if (timeline) + fps = timeline->info.fps; + else if (clip && clip->Reader()) + fps = clip->Reader()->info.fps; + double fps_d = fps.ToDouble(); + double t = fps_d > 0 ? frame_number / fps_d : frame_number; + + const float k_track = tracking.GetValue(frame_number); + const float k_bleed = bleed.GetValue(frame_number); + const float k_soft = softness.GetValue(frame_number); + const float k_noise = noise.GetValue(frame_number); + const float k_stripe = stripe.GetValue(frame_number); + const float k_bands = staticBands.GetValue(frame_number); + + int r_y = std::round(lerp(0.0f, 2.0f, k_soft)); + if (k_noise > 0.6f) + r_y = std::min(r_y, 1); + if (r_y > 0) { +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) + box_blur_row(&Y[y * w], &tmpY[y * w], w, r_y); + Y.swap(tmpY); + } + + float shift = lerp(0.0f, 2.5f, k_bleed); + int r_c = std::round(lerp(0.0f, 3.0f, k_bleed)); + float sat = 1.0f - 0.30f * k_bleed; + float shift_h = shift * 0.5f; +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) { + const float *srcU = &U[y * Uw]; + const float *srcV = &V[y * Uw]; + float *dstU = &tmpU[y * Uw]; + float *dstV = &tmpV[y * Uw]; + for (int x = 0; x < Uw; ++x) { + float xs = std::clamp(x - shift_h, 0.0f, float(Uw - 1)); + int x0 = int(xs); + int x1 = std::min(x0 + 1, Uw - 1); + float t = xs - x0; + dstU[x] = srcU[x0] * (1 - t) + srcU[x1] * t; + dstV[x] = srcV[x0] * (1 - t) + srcV[x1] * t; + } + } + U.swap(tmpU); + V.swap(tmpV); + + if (r_c > 0) { +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) + box_blur_row(&U[y * Uw], &tmpU[y * Uw], Uw, r_c); + U.swap(tmpU); +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) + box_blur_row(&V[y * Uw], &tmpV[y * Uw], Uw, r_c); + V.swap(tmpV); + } + + uint32_t SEED = fnv1a_32(Id()) ^ (uint32_t)seed_offset; + uint32_t schedSalt = (uint32_t)(k_bands * 64.0f) ^ + ((uint32_t)(k_stripe * 64.0f) << 8) ^ + ((uint32_t)(k_noise * 64.0f) << 16); + uint32_t SCHED_SEED = SEED ^ fnv1a_32(schedSalt, 0x9e3779b9u); + const float PI = 3.14159265358979323846f; + + float sigmaY = lerp(0.0f, 0.08f, k_noise); + const float decay = 0.88f + 0.08f * k_noise; + const float amp = 0.18f * k_noise; + const float baseP = 0.0025f + 0.02f * k_noise; + + float Hfixed = lerp(0.0f, 0.12f * h, k_stripe); + float Gfixed = 0.10f * k_stripe; + float Nfixed = 1.0f + 1.5f * k_stripe; + + float rate = 0.4f * k_bands; + int dur_frames = std::round(lerp(1.0f, 6.0f, k_bands)); + float Hburst = lerp(0.06f * h, 0.25f * h, k_bands); + float Gburst = lerp(0.10f, 0.25f, k_bands); + float sat_band = lerp(0.8f, 0.5f, k_bands); + float Nburst = 1.0f + 2.0f * k_bands; + + struct Band { float center; double t0; }; + std::vector bands; + if (k_bands > 0.0f && rate > 0.0f) { + const double win_len = 0.25; + int win_idx = int(t / win_len); + double lambda = rate * win_len * + (0.25 + 1.5f * row_density(SCHED_SEED, frame_number, 0)); + double prob_ge1 = 1.0 - std::exp(-lambda); + double prob_ge2 = 1.0 - std::exp(-lambda) - lambda * std::exp(-lambda); + + auto spawn_band = [&](int kseed) { + float r1 = hash01(SCHED_SEED, uint32_t(win_idx), 11 + kseed, 0); + float start = r1 * win_len; + float center = + hash01(SCHED_SEED, uint32_t(win_idx), 12 + kseed, 0) * (h - Hburst) + + 0.5f * Hburst; + double t0 = win_idx * win_len + start; + double t1 = t0 + dur_frames / (fps_d > 0 ? fps_d : 1.0); + if (t >= t0 && t < t1) + bands.push_back({center, t0}); + }; + + float r = hash01(SCHED_SEED, uint32_t(win_idx), 9, 0); + if (r < prob_ge1) + spawn_band(0); + if (r < prob_ge2) + spawn_band(1); + } + + float ft = 2.0f; + int kf = int(std::floor(t * ft)); + float a = float(t * ft - kf); + +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) { + float bandF = 0.0f; + if (Hfixed > 0.0f && y >= h - Hfixed) + bandF = (y - (h - Hfixed)) / std::max(1.0f, Hfixed); + float burstF = 0.0f; + for (const auto &b : bands) { + float halfH = Hburst * 0.5f; + float dist = std::abs(y - b.center); + float profile = std::max(0.0f, 1.0f - dist / halfH); + float life = float((t - b.t0) * fps_d); + float env = (life < 1.0f) + ? life + : (life < dur_frames - 1 ? 1.0f + : std::max(0.0f, dur_frames - life)); + burstF = std::max(burstF, profile * env); + } + + float sat_row = 1.0f - (1.0f - sat_band) * burstF; + if (burstF > 0.0f && sat_row != 1.0f) { + float *urow = &U[y * Uw]; + float *vrow = &V[y * Uw]; + for (int xh = 0; xh < Uw; ++xh) { + urow[xh] *= sat_row; + vrow[xh] *= sat_row; + } + } + + float rowBias = row_density(SEED, frame_number, y); + float p = baseP * (0.25f + 1.5f * rowBias); + p *= (1.0f + 1.5f * bandF + 2.0f * burstF); + + float hum = 0.008f * k_noise * + std::sin(2 * PI * (y * (6.0f / h) + 0.08f * t)); + uint32_t s0 = SEED ^ 0x9e37u * kf ^ 0x85ebu * y; + uint32_t s1 = SEED ^ 0x9e37u * (kf + 1) ^ 0x85ebu * y ^ 0x1234567u; + auto step = [](uint32_t &s) { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + return s; + }; + float lift = Gfixed * bandF + Gburst * burstF; + float rowSigma = sigmaY * (1 + (Nfixed - 1) * bandF + + (Nburst - 1) * burstF); + float k = 0.15f + 0.35f * hash01(SEED, uint32_t(frame_number), y, 777); + float sL = 0.0f, sR = 0.0f; + for (int x = 0; x < w; ++x) { + if (hash01(SEED, uint32_t(frame_number), y, x) < p) + sL = 1.0f; + if (hash01(SEED, uint32_t(frame_number), y, w - 1 - x) < p * 0.7f) + sR = 1.0f; + float n = ((step(s0) & 0xFFFFFF) / 16777215.0f) * (1 - a) + + ((step(s1) & 0xFFFFFF) / 16777215.0f) * a; + int idx = y * w + x; + float mt = std::clamp((Y[idx] - 0.2f) / (0.8f - 0.2f), 0.0f, 1.0f); + float val = Y[idx] + lift + rowSigma * (2 * n - 1) * + (0.6f + 0.4f * mt) + hum; + float streak = amp * (sL + sR); + float newY = val + streak * (k + (1.0f - val)); + Y[idx] = std::clamp(newY, 0.0f, 1.0f); + sL *= decay; + sR *= decay; + } + } + + float A = lerp(0.0f, 3.0f, k_track); // pixels + float f = lerp(0.25f, 1.2f, k_track); // Hz + float Hsk = lerp(0.0f, 0.10f * h, k_track); // pixels + float S = lerp(0.0f, 5.0f, k_track); // pixels + float phase = 2 * PI * (f * t) + 0.7f * (SEED * 0.001f); + for (int y = 0; y < h; ++y) { + float base = A * std::sin(2 * PI * 0.0035f * y + phase); + float skew = (y >= h - Hsk) + ? S * ((y - (h - Hsk)) / std::max(1.0f, Hsk)) + : 0.0f; + dx[y] = base + skew; + } + + auto remap_line = [&](const float *src, float *dst, int width, float scale) { +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) { + float off = dx[y] * scale; + const float *s = src + y * width; + float *d = dst + y * width; + int start = std::max(0, int(std::ceil(-off))); + int end = std::min(width, int(std::floor(width - off))); + float xs = start + off; + int x0 = int(xs); + float t = xs - x0; + for (int x = start; x < end; ++x) { + int x1 = x0 + 1; + d[x] = s[x0] * (1 - t) + s[x1] * t; + xs += 1.0f; + x0 = int(xs); + t = xs - x0; + } + for (int x = 0; x < start; ++x) + d[x] = s[0]; + for (int x = end; x < width; ++x) + d[x] = s[width - 1]; + } + }; + + remap_line(Y.data(), tmpY.data(), w, 1.0f); + Y.swap(tmpY); + remap_line(U.data(), tmpU.data(), Uw, 0.5f); + U.swap(tmpU); + remap_line(V.data(), tmpV.data(), Uw, 0.5f); + V.swap(tmpV); + +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int y = 0; y < h; ++y) { + float *yrow = &Y[y * w]; + float *urow = &U[y * Uw]; + float *vrow = &V[y * Uw]; + uint32_t *row = base + y * stride; + for (int x = 0; x < w; ++x) { + float xs = x * 0.5f; + int x0 = int(xs); + int x1 = std::min(x0 + 1, Uw - 1); + float t = xs - x0; + float u = (urow[x0] * (1 - t) + urow[x1] * t) * sat; + float v = (vrow[x0] * (1 - t) + vrow[x1] * t) * sat; + float yv = yrow[x]; + float r = yv + 1.13983f * v; + float g = yv - 0.39465f * u - 0.58060f * v; + float b = yv + 2.03211f * u; + int R = int(std::clamp(r, 0.0f, 1.0f) * 255.0f); + int G = int(std::clamp(g, 0.0f, 1.0f) * 255.0f); + int B = int(std::clamp(b, 0.0f, 1.0f) * 255.0f); + uint32_t A = row[x] & 0xFF000000u; + row[x] = A | (R << 16) | (G << 8) | B; + } + } + + return frame; +} + +// JSON +std::string AnalogTape::Json() const { return JsonValue().toStyledString(); } + +Json::Value AnalogTape::JsonValue() const { + Json::Value root = EffectBase::JsonValue(); + root["type"] = info.class_name; + root["tracking"] = tracking.JsonValue(); + root["bleed"] = bleed.JsonValue(); + root["softness"] = softness.JsonValue(); + root["noise"] = noise.JsonValue(); + root["stripe"] = stripe.JsonValue(); + root["static_bands"] = staticBands.JsonValue(); + root["seed_offset"] = seed_offset; + return root; +} + +void AnalogTape::SetJson(const std::string value) { + try { + Json::Value root = openshot::stringToJson(value); + SetJsonValue(root); + } catch (const std::exception &) { + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +void AnalogTape::SetJsonValue(const Json::Value root) { + EffectBase::SetJsonValue(root); + if (!root["tracking"].isNull()) + tracking.SetJsonValue(root["tracking"]); + if (!root["bleed"].isNull()) + bleed.SetJsonValue(root["bleed"]); + if (!root["softness"].isNull()) + softness.SetJsonValue(root["softness"]); + if (!root["noise"].isNull()) + noise.SetJsonValue(root["noise"]); + if (!root["stripe"].isNull()) + stripe.SetJsonValue(root["stripe"]); + if (!root["static_bands"].isNull()) + staticBands.SetJsonValue(root["static_bands"]); + if (!root["seed_offset"].isNull()) + seed_offset = root["seed_offset"].asInt(); +} + +std::string AnalogTape::PropertiesJSON(int64_t requested_frame) const { + Json::Value root = BasePropertiesJSON(requested_frame); + root["tracking"] = + add_property_json("Tracking", tracking.GetValue(requested_frame), "float", + "", &tracking, 0, 1, false, requested_frame); + root["bleed"] = + add_property_json("Bleed", bleed.GetValue(requested_frame), "float", "", + &bleed, 0, 1, false, requested_frame); + root["softness"] = + add_property_json("Softness", softness.GetValue(requested_frame), "float", + "", &softness, 0, 1, false, requested_frame); + root["noise"] = + add_property_json("Noise", noise.GetValue(requested_frame), "float", "", + &noise, 0, 1, false, requested_frame); + root["stripe"] = + add_property_json("Stripe", stripe.GetValue(requested_frame), "float", + "Bottom tracking stripe brightness and noise.", + &stripe, 0, 1, false, requested_frame); + root["static_bands"] = + add_property_json("Static Bands", staticBands.GetValue(requested_frame), + "float", + "Short bright static bands and extra dropouts.", + &staticBands, 0, 1, false, requested_frame); + root["seed_offset"] = + add_property_json("Seed Offset", seed_offset, "int", "", NULL, 0, 1000, + false, requested_frame); + return root.toStyledString(); +} diff --git a/src/effects/AnalogTape.h b/src/effects/AnalogTape.h new file mode 100644 index 000000000..1c2874461 --- /dev/null +++ b/src/effects/AnalogTape.h @@ -0,0 +1,136 @@ +/** + * @file + * @brief Header file for AnalogTape effect class + * + * Vintage home video wobble, bleed, and grain. + * + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_ANALOGTAPE_EFFECT_H +#define OPENSHOT_ANALOGTAPE_EFFECT_H + +#include "../EffectBase.h" +#include "../Frame.h" +#include "../Json.h" +#include "../KeyFrame.h" + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define OS_RESTRICT __restrict__ +#else +#define OS_RESTRICT +#endif + +namespace openshot { + +/// Analog video tape simulation effect. +class AnalogTape : public EffectBase { +private: + void init_effect_details(); + static inline uint32_t fnv1a_32(const std::string &s) { + uint32_t h = 2166136261u; + for (unsigned char c : s) { + h ^= c; + h *= 16777619u; + } + return h; + } + static inline uint32_t fnv1a_32(uint32_t h, uint32_t d) { + unsigned char bytes[4]; + bytes[0] = d & 0xFF; + bytes[1] = (d >> 8) & 0xFF; + bytes[2] = (d >> 16) & 0xFF; + bytes[3] = (d >> 24) & 0xFF; + for (int i = 0; i < 4; ++i) { + h ^= bytes[i]; + h *= 16777619u; + } + return h; + } + static inline float hash01(uint32_t seed, uint32_t a, uint32_t b, uint32_t c) { + uint32_t h = fnv1a_32(seed, a); + h = fnv1a_32(h, b); + h = fnv1a_32(h, c); + return h / 4294967295.0f; + } + static inline float row_density(uint32_t seed, int frame, int y) { + int tc = (frame >> 3); + int y0 = (y >> 3); + float a = (y & 7) / 8.0f; + float h0 = hash01(seed, tc, y0, 31); + float h1 = hash01(seed, tc, y0 + 1, 31); + float m = (1 - a) * h0 + a * h1; + return m * m; + } + static inline void box_blur_row(const float *OS_RESTRICT src, + float *OS_RESTRICT dst, int w, int r) { + if (r == 0) { + std::memcpy(dst, src, w * sizeof(float)); + return; + } + const int win = 2 * r + 1; + float sum = 0.0f; + for (int k = -r; k <= r; ++k) + sum += src[std::clamp(k, 0, w - 1)]; + dst[0] = sum / win; + for (int x = 1; x < w; ++x) { + int add = std::min(w - 1, x + r); + int sub = std::max(0, x - r - 1); + sum += src[add] - src[sub]; + dst[x] = sum / win; + } + } + + int last_w = 0, last_h = 0; + std::vector Y, U, V, tmpY, tmpU, tmpV, dx; + +public: + Keyframe tracking; ///< tracking wobble amount + Keyframe bleed; ///< color bleed amount + Keyframe softness; ///< luma blur radius + Keyframe noise; ///< grain/dropouts amount + Keyframe stripe; ///< bottom tracking stripe strength + Keyframe staticBands; ///< burst static band strength + int seed_offset; ///< seed offset for deterministic randomness + + AnalogTape(); + AnalogTape(Keyframe tracking, Keyframe bleed, Keyframe softness, + Keyframe noise, Keyframe stripe, Keyframe staticBands, + int seed_offset = 0); + + std::shared_ptr + GetFrame(std::shared_ptr frame, + int64_t frame_number) override; + + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); + } + + // JSON + std::string Json() const override; + void SetJson(const std::string value) override; + Json::Value JsonValue() const override; + void SetJsonValue(const Json::Value root) override; + +std::string PropertiesJSON(int64_t requested_frame) const override; +}; + +} // namespace openshot + +#undef OS_RESTRICT + +#endif diff --git a/src/effects/ColorMap.cpp b/src/effects/ColorMap.cpp index 3587cd9d3..ef735f427 100644 --- a/src/effects/ColorMap.cpp +++ b/src/effects/ColorMap.cpp @@ -12,6 +12,7 @@ #include "ColorMap.h" #include "Exceptions.h" +#include #include #include @@ -22,12 +23,16 @@ void ColorMap::load_cube_file() if (lut_path.empty()) { lut_data.clear(); lut_size = 0; + lut_type = LUTType::None; needs_refresh = false; return; } int parsed_size = 0; std::vector parsed_data; + bool parsed_is_3d = false; + std::array parsed_domain_min{0.0f, 0.0f, 0.0f}; + std::array parsed_domain_max{1.0f, 1.0f, 1.0f}; #pragma omp critical(load_lut) { @@ -36,46 +41,81 @@ void ColorMap::load_cube_file() // leave parsed_size == 0 } else { QTextStream in(&file); - QString line; QRegularExpression ws_re("\\s+"); - - // 1) Find LUT_3D_SIZE - while (!in.atEnd()) { - line = in.readLine().trimmed(); - if (line.startsWith("LUT_3D_SIZE")) { - auto parts = line.split(ws_re); - if (parts.size() >= 2) { - parsed_size = parts[1].toInt(); + auto parse_domain_line = [&](const QString &line) { + if (!line.startsWith("DOMAIN_MIN") && !line.startsWith("DOMAIN_MAX")) + return; + auto parts = line.split(ws_re); + if (parts.size() < 4) + return; + auto assign_values = [&](std::array &target) { + target[0] = parts[1].toFloat(); + target[1] = parts[2].toFloat(); + target[2] = parts[3].toFloat(); + }; + if (line.startsWith("DOMAIN_MIN")) + assign_values(parsed_domain_min); + else + assign_values(parsed_domain_max); + }; + + auto try_parse = [&](const QString &keyword, bool want3d) -> bool { + if (!file.seek(0) || !in.seek(0)) + return false; + + QString line; + int detected_size = 0; + while (!in.atEnd()) { + line = in.readLine().trimmed(); + parse_domain_line(line); + if (line.startsWith(keyword)) { + auto parts = line.split(ws_re); + if (parts.size() >= 2) { + detected_size = parts[1].toInt(); + } + break; } - break; } - } - - // 2) Read N³ lines of R G B floats - if (parsed_size > 0) { - int total = parsed_size * parsed_size * parsed_size; - parsed_data.reserve(size_t(total * 3)); - while (!in.atEnd() && int(parsed_data.size()) < total * 3) { + if (detected_size <= 0) + return false; + + const int total_entries = want3d + ? detected_size * detected_size * detected_size + : detected_size; + std::vector data; + data.reserve(size_t(total_entries * 3)); + while (!in.atEnd() && int(data.size()) < total_entries * 3) { line = in.readLine().trimmed(); if (line.isEmpty() || line.startsWith("#") || - line.startsWith("TITLE") || - line.startsWith("DOMAIN")) + line.startsWith("TITLE")) + { + continue; + } + if (line.startsWith("DOMAIN_MIN") || + line.startsWith("DOMAIN_MAX")) { + parse_domain_line(line); continue; } auto vals = line.split(ws_re); if (vals.size() >= 3) { - // .cube file is R G B - parsed_data.push_back(vals[0].toFloat()); - parsed_data.push_back(vals[1].toFloat()); - parsed_data.push_back(vals[2].toFloat()); + data.push_back(vals[0].toFloat()); + data.push_back(vals[1].toFloat()); + data.push_back(vals[2].toFloat()); } } - if (int(parsed_data.size()) != total * 3) { - parsed_data.clear(); - parsed_size = 0; - } + if (int(data.size()) != total_entries * 3) + return false; + + parsed_size = detected_size; + parsed_is_3d = want3d; + parsed_data.swap(data); + return true; + }; + + if (!try_parse("LUT_3D_SIZE", true)) { + try_parse("LUT_1D_SIZE", false); } } } @@ -83,9 +123,15 @@ void ColorMap::load_cube_file() if (parsed_size > 0) { lut_size = parsed_size; lut_data.swap(parsed_data); + lut_type = parsed_is_3d ? LUTType::LUT3D : LUTType::LUT1D; + lut_domain_min = parsed_domain_min; + lut_domain_max = parsed_domain_max; } else { lut_data.clear(); lut_size = 0; + lut_type = LUTType::None; + lut_domain_min = std::array{0.0f, 0.0f, 0.0f}; + lut_domain_max = std::array{1.0f, 1.0f, 1.0f}; } needs_refresh = false; } @@ -101,7 +147,8 @@ void ColorMap::init_effect_details() } ColorMap::ColorMap() - : lut_path(""), lut_size(0), needs_refresh(true), + : lut_path(""), lut_size(0), lut_type(LUTType::None), needs_refresh(true), + lut_domain_min{0.0f, 0.0f, 0.0f}, lut_domain_max{1.0f, 1.0f, 1.0f}, intensity(1.0), intensity_r(1.0), intensity_g(1.0), intensity_b(1.0) { init_effect_details(); @@ -115,7 +162,9 @@ ColorMap::ColorMap(const std::string &path, const Keyframe &iB) : lut_path(path), lut_size(0), + lut_type(LUTType::None), needs_refresh(true), + lut_domain_min{0.0f, 0.0f, 0.0f}, lut_domain_max{1.0f, 1.0f, 1.0f}, intensity(i), intensity_r(iR), intensity_g(iG), @@ -134,7 +183,7 @@ ColorMap::GetFrame(std::shared_ptr frame, int64_t frame_number) needs_refresh = false; } - if (lut_data.empty()) + if (lut_data.empty() || lut_size <= 0 || lut_type == LUTType::None) return frame; auto image = frame->GetImage(); @@ -146,6 +195,28 @@ ColorMap::GetFrame(std::shared_ptr frame, int64_t frame_number) float tG = float(intensity_g.GetValue(frame_number)) * overall; float tB = float(intensity_b.GetValue(frame_number)) * overall; + const bool use3d = (lut_type == LUTType::LUT3D); + const bool use1d = (lut_type == LUTType::LUT1D); + const int lut_dim = lut_size; + const std::vector &table = lut_data; + const int data_count = int(table.size()); + + auto sample1d = [&](float value, int channel) -> float { + if (lut_dim <= 1) { + int base = std::min(channel, data_count - 1); + return table[base]; + } + float scaled = value * float(lut_dim - 1); + int i0 = int(floor(scaled)); + int i1 = std::min(i0 + 1, lut_dim - 1); + float t = scaled - i0; + int base0 = std::max(0, std::min(i0 * 3 + channel, data_count - 1)); + int base1 = std::max(0, std::min(i1 * 3 + channel, data_count - 1)); + float v0 = table[base0]; + float v1 = table[base1]; + return v0 * (1.0f - t) + v1 * t; + }; + int pixel_count = w * h; #pragma omp parallel for for (int i = 0; i < pixel_count; ++i) { @@ -164,56 +235,73 @@ ColorMap::GetFrame(std::shared_ptr frame, int64_t frame_number) float Gn = G * (1.0f / 255.0f); float Bn = B * (1.0f / 255.0f); - // map into LUT space [0 .. size-1] - float rf = Rn * (lut_size - 1); - float gf = Gn * (lut_size - 1); - float bf = Bn * (lut_size - 1); - - int r0 = int(floor(rf)), r1 = std::min(r0 + 1, lut_size - 1); - int g0 = int(floor(gf)), g1 = std::min(g0 + 1, lut_size - 1); - int b0 = int(floor(bf)), b1 = std::min(b0 + 1, lut_size - 1); - - float dr = rf - r0; - float dg = gf - g0; - float db = bf - b0; - - // compute base offsets with red fastest, then green, then blue - int base000 = ((b0 * lut_size + g0) * lut_size + r0) * 3; - int base100 = ((b0 * lut_size + g0) * lut_size + r1) * 3; - int base010 = ((b0 * lut_size + g1) * lut_size + r0) * 3; - int base110 = ((b0 * lut_size + g1) * lut_size + r1) * 3; - int base001 = ((b1 * lut_size + g0) * lut_size + r0) * 3; - int base101 = ((b1 * lut_size + g0) * lut_size + r1) * 3; - int base011 = ((b1 * lut_size + g1) * lut_size + r0) * 3; - int base111 = ((b1 * lut_size + g1) * lut_size + r1) * 3; - - // trilinear interpolation - // red - float c00 = lut_data[base000 + 0] * (1 - dr) + lut_data[base100 + 0] * dr; - float c01 = lut_data[base001 + 0] * (1 - dr) + lut_data[base101 + 0] * dr; - float c10 = lut_data[base010 + 0] * (1 - dr) + lut_data[base110 + 0] * dr; - float c11 = lut_data[base011 + 0] * (1 - dr) + lut_data[base111 + 0] * dr; - float c0 = c00 * (1 - dg) + c10 * dg; - float c1 = c01 * (1 - dg) + c11 * dg; - float lr = c0 * (1 - db) + c1 * db; - - // green - c00 = lut_data[base000 + 1] * (1 - dr) + lut_data[base100 + 1] * dr; - c01 = lut_data[base001 + 1] * (1 - dr) + lut_data[base101 + 1] * dr; - c10 = lut_data[base010 + 1] * (1 - dr) + lut_data[base110 + 1] * dr; - c11 = lut_data[base011 + 1] * (1 - dr) + lut_data[base111 + 1] * dr; - c0 = c00 * (1 - dg) + c10 * dg; - c1 = c01 * (1 - dg) + c11 * dg; - float lg = c0 * (1 - db) + c1 * db; - - // blue - c00 = lut_data[base000 + 2] * (1 - dr) + lut_data[base100 + 2] * dr; - c01 = lut_data[base001 + 2] * (1 - dr) + lut_data[base101 + 2] * dr; - c10 = lut_data[base010 + 2] * (1 - dr) + lut_data[base110 + 2] * dr; - c11 = lut_data[base011 + 2] * (1 - dr) + lut_data[base111 + 2] * dr; - c0 = c00 * (1 - dg) + c10 * dg; - c1 = c01 * (1 - dg) + c11 * dg; - float lb = c0 * (1 - db) + c1 * db; + auto normalize_to_domain = [&](float value, int channel) -> float { + float min_val = lut_domain_min[channel]; + float max_val = lut_domain_max[channel]; + float range = max_val - min_val; + if (range <= 0.0f) + return std::clamp(value, 0.0f, 1.0f); + float normalized = (value - min_val) / range; + return std::clamp(normalized, 0.0f, 1.0f); + }; + float Rdn = normalize_to_domain(Rn, 0); + float Gdn = normalize_to_domain(Gn, 1); + float Bdn = normalize_to_domain(Bn, 2); + + float lr = Rn; + float lg = Gn; + float lb = Bn; + + if (use3d) { + float rf = Rdn * (lut_dim - 1); + float gf = Gdn * (lut_dim - 1); + float bf = Bdn * (lut_dim - 1); + + int r0 = int(floor(rf)), r1 = std::min(r0 + 1, lut_dim - 1); + int g0 = int(floor(gf)), g1 = std::min(g0 + 1, lut_dim - 1); + int b0 = int(floor(bf)), b1 = std::min(b0 + 1, lut_dim - 1); + + float dr = rf - r0; + float dg = gf - g0; + float db = bf - b0; + + int base000 = ((b0 * lut_dim + g0) * lut_dim + r0) * 3; + int base100 = ((b0 * lut_dim + g0) * lut_dim + r1) * 3; + int base010 = ((b0 * lut_dim + g1) * lut_dim + r0) * 3; + int base110 = ((b0 * lut_dim + g1) * lut_dim + r1) * 3; + int base001 = ((b1 * lut_dim + g0) * lut_dim + r0) * 3; + int base101 = ((b1 * lut_dim + g0) * lut_dim + r1) * 3; + int base011 = ((b1 * lut_dim + g1) * lut_dim + r0) * 3; + int base111 = ((b1 * lut_dim + g1) * lut_dim + r1) * 3; + + float c00 = table[base000 + 0] * (1 - dr) + table[base100 + 0] * dr; + float c01 = table[base001 + 0] * (1 - dr) + table[base101 + 0] * dr; + float c10 = table[base010 + 0] * (1 - dr) + table[base110 + 0] * dr; + float c11 = table[base011 + 0] * (1 - dr) + table[base111 + 0] * dr; + float c0 = c00 * (1 - dg) + c10 * dg; + float c1 = c01 * (1 - dg) + c11 * dg; + lr = c0 * (1 - db) + c1 * db; + + c00 = table[base000 + 1] * (1 - dr) + table[base100 + 1] * dr; + c01 = table[base001 + 1] * (1 - dr) + table[base101 + 1] * dr; + c10 = table[base010 + 1] * (1 - dr) + table[base110 + 1] * dr; + c11 = table[base011 + 1] * (1 - dr) + table[base111 + 1] * dr; + c0 = c00 * (1 - dg) + c10 * dg; + c1 = c01 * (1 - dg) + c11 * dg; + lg = c0 * (1 - db) + c1 * db; + + c00 = table[base000 + 2] * (1 - dr) + table[base100 + 2] * dr; + c01 = table[base001 + 2] * (1 - dr) + table[base101 + 2] * dr; + c10 = table[base010 + 2] * (1 - dr) + table[base110 + 2] * dr; + c11 = table[base011 + 2] * (1 - dr) + table[base111 + 2] * dr; + c0 = c00 * (1 - dg) + c10 * dg; + c1 = c01 * (1 - dg) + c11 * dg; + lb = c0 * (1 - db) + c1 * db; + } else if (use1d) { + lr = sample1d(Rdn, 0); + lg = sample1d(Gdn, 1); + lb = sample1d(Bdn, 2); + } // blend per-channel, re-premultiply alpha float outR = (lr * tR + Rn * (1 - tR)) * alpha; diff --git a/src/effects/ColorMap.h b/src/effects/ColorMap.h index 91cc73868..61525fb93 100644 --- a/src/effects/ColorMap.h +++ b/src/effects/ColorMap.h @@ -21,23 +21,29 @@ #include #include #include +#include namespace openshot { /** - * @brief Applies a 3D LUT (.cube) color transform to each frame. + * @brief Applies a 1D or 3D LUT (.cube) color transform to each frame. * - * Loads a .cube file (LUT_3D_SIZE N × N × N) into memory, then for each pixel - * uses nearest‐neighbor lookup and blends the result by keyframable per‐channel intensities. + * Loads a .cube file (supporting LUT_1D_SIZE and LUT_3D_SIZE) into memory, then for each pixel + * interpolates the lookup value and blends the result by keyframable per‐channel intensities. */ class ColorMap : public EffectBase { private: + enum class LUTType { None, LUT1D, LUT3D }; + std::string lut_path; ///< Filesystem path to .cube LUT file - int lut_size; ///< Dimension N of the cube (LUT_3D_SIZE) - std::vector lut_data; ///< Flat array [N³ × 3] RGB lookup table + int lut_size; ///< Dimension of LUT (entries for 1D, cube edge for 3D) + std::vector lut_data; ///< Flat array containing LUT entries + LUTType lut_type; ///< Indicates if LUT is 1D or 3D bool needs_refresh; ///< Reload LUT on next frame + std::array lut_domain_min; ///< Input domain minimum per channel + std::array lut_domain_max; ///< Input domain maximum per channel /// Populate info fields (class_name, name, description) void init_effect_details(); diff --git a/src/effects/CropHelpers.cpp b/src/effects/CropHelpers.cpp new file mode 100644 index 000000000..4b5229e1c --- /dev/null +++ b/src/effects/CropHelpers.cpp @@ -0,0 +1,64 @@ +/** + * @file + * @brief Shared helpers for Crop effect scaling logic + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "CropHelpers.h" + +#include +#include +#include + +#include "../Clip.h" +#include "Crop.h" + +namespace openshot { + +const Crop* FindResizingCropEffect(Clip* clip) { + if (!clip) { + return nullptr; + } + + for (auto effect : clip->Effects()) { + if (auto crop_effect = dynamic_cast(effect)) { + if (crop_effect->resize) { + return crop_effect; + } + } + } + + return nullptr; +} + +void ApplyCropResizeScale(Clip* clip, int source_width, int source_height, int& max_width, int& max_height) { + const Crop* crop_effect = FindResizingCropEffect(clip); + if (!crop_effect) { + return; + } + + const float max_left = crop_effect->left.GetMaxPoint().co.Y; + const float max_right = crop_effect->right.GetMaxPoint().co.Y; + const float max_top = crop_effect->top.GetMaxPoint().co.Y; + const float max_bottom = crop_effect->bottom.GetMaxPoint().co.Y; + + const float visible_width = std::max(0.01f, 1.0f - max_left - max_right); + const float visible_height = std::max(0.01f, 1.0f - max_top - max_bottom); + + const double scaled_width = std::ceil(max_width / visible_width); + const double scaled_height = std::ceil(max_height / visible_height); + + const double clamped_width = std::min(source_width, scaled_width); + const double clamped_height = std::min(source_height, scaled_height); + + max_width = static_cast(std::min(std::numeric_limits::max(), clamped_width)); + max_height = static_cast(std::min(std::numeric_limits::max(), clamped_height)); +} + +} // namespace openshot diff --git a/src/effects/CropHelpers.h b/src/effects/CropHelpers.h new file mode 100644 index 000000000..5e47a21b1 --- /dev/null +++ b/src/effects/CropHelpers.h @@ -0,0 +1,29 @@ +/** + * @file + * @brief Shared helpers for Crop effect scaling logic + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_CROP_HELPERS_H +#define OPENSHOT_CROP_HELPERS_H + +namespace openshot { + +class Clip; +class Crop; + +/// Return the first Crop effect on this clip that has resize enabled (if any) +const Crop* FindResizingCropEffect(Clip* clip); + +/// Scale the requested max_width / max_height based on the Crop resize amount, capped by source size +void ApplyCropResizeScale(Clip* clip, int source_width, int source_height, int& max_width, int& max_height); + +} // namespace openshot + +#endif // OPENSHOT_CROP_HELPERS_H diff --git a/src/effects/ObjectDetection.cpp b/src/effects/ObjectDetection.cpp index 6bc019552..6e1ae97c3 100644 --- a/src/effects/ObjectDetection.cpp +++ b/src/effects/ObjectDetection.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "effects/ObjectDetection.h" #include "effects/Tracker.h" @@ -29,29 +30,16 @@ using namespace std; using namespace openshot; -/// Blank constructor, useful when using Json to load the effect properties -ObjectDetection::ObjectDetection(std::string clipObDetectDataPath) : -display_box_text(1.0), display_boxes(1.0) -{ - // Init effect properties - init_effect_details(); - - // Tries to load the tracker data from protobuf - LoadObjDetectdData(clipObDetectDataPath); - - // Initialize the selected object index as the first object index - selectedObjectIndex = trackedObjects.begin()->first; -} - // Default constructor -ObjectDetection::ObjectDetection() : - display_box_text(1.0), display_boxes(1.0) +ObjectDetection::ObjectDetection() + : display_box_text(1.0) + , display_boxes(1.0) { - // Init effect properties + // Init effect metadata init_effect_details(); - // Initialize the selected object index as the first object index - selectedObjectIndex = trackedObjects.begin()->first; + // We haven’t loaded any protobuf yet, so there's nothing to pick. + selectedObjectIndex = -1; } // Init effect settings @@ -167,108 +155,99 @@ std::shared_ptr ObjectDetection::GetFrame(std::shared_ptr frame, i } // Load protobuf data file -bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath){ - // Create tracker message - pb_objdetect::ObjDetect objMessage; - - // Read the existing tracker message. - std::fstream input(inputFilePath, std::ios::in | std::ios::binary); - if (!objMessage.ParseFromIstream(&input)) { - std::cerr << "Failed to parse protobuf message." << std::endl; - return false; - } - - // Make sure classNames, detectionsData and trackedObjects are empty - classNames.clear(); - detectionsData.clear(); - trackedObjects.clear(); - - // Seed to generate same random numbers - std::srand(1); - // Get all classes names and assign a color to them - for(int i = 0; i < objMessage.classnames_size(); i++) - { - classNames.push_back(objMessage.classnames(i)); - classesColor.push_back(cv::Scalar(std::rand()%205 + 50, std::rand()%205 + 50, std::rand()%205 + 50)); - } - - // Iterate over all frames of the saved message - for (size_t i = 0; i < objMessage.frame_size(); i++) - { - // Create protobuf message reader - const pb_objdetect::Frame& pbFrameData = objMessage.frame(i); - - // Get frame Id - size_t id = pbFrameData.id(); - - // Load bounding box data - const google::protobuf::RepeatedPtrField &pBox = pbFrameData.bounding_box(); +bool ObjectDetection::LoadObjDetectdData(std::string inputFilePath) +{ + // Parse the file + pb_objdetect::ObjDetect objMessage; + std::fstream input(inputFilePath, std::ios::in | std::ios::binary); + if (!objMessage.ParseFromIstream(&input)) { + std::cerr << "Failed to parse protobuf message." << std::endl; + return false; + } - // Construct data vectors related to detections in the current frame - std::vector classIds; - std::vector confidences; - std::vector> boxes; - std::vector objectIds; + // Clear out any old state + classNames.clear(); + detectionsData.clear(); + trackedObjects.clear(); + + // Seed colors for each class + std::srand(1); + for (int i = 0; i < objMessage.classnames_size(); ++i) { + classNames.push_back(objMessage.classnames(i)); + classesColor.push_back(cv::Scalar( + std::rand() % 205 + 50, + std::rand() % 205 + 50, + std::rand() % 205 + 50 + )); + } - // Iterate through the detected objects - for(int i = 0; i < pbFrameData.bounding_box_size(); i++) - { - // Get bounding box coordinates - float x = pBox.Get(i).x(); - float y = pBox.Get(i).y(); - float w = pBox.Get(i).w(); - float h = pBox.Get(i).h(); - // Get class Id (which will be assign to a class name) - int classId = pBox.Get(i).classid(); - // Get prediction confidence - float confidence = pBox.Get(i).confidence(); - - // Get the object Id - int objectId = pBox.Get(i).objectid(); - - // Search for the object id on trackedObjects map - auto trackedObject = trackedObjects.find(objectId); - // Check if object already exists on the map - if (trackedObject != trackedObjects.end()) - { - // Add a new BBox to it - trackedObject->second->AddBox(id, x+(w/2), y+(h/2), w, h, 0.0); - } - else - { - // There is no tracked object with that id, so insert a new one - TrackedObjectBBox trackedObj((int)classesColor[classId](0), (int)classesColor[classId](1), (int)classesColor[classId](2), (int)0); - trackedObj.stroke_alpha = Keyframe(1.0); - trackedObj.AddBox(id, x+(w/2), y+(h/2), w, h, 0.0); - - std::shared_ptr trackedObjPtr = std::make_shared(trackedObj); - ClipBase* parentClip = this->ParentClip(); - trackedObjPtr->ParentClip(parentClip); - - // Create a temp ID. This ID is necessary to initialize the object_id Json list - // this Id will be replaced by the one created in the UI - trackedObjPtr->Id(std::to_string(objectId)); - trackedObjects.insert({objectId, trackedObjPtr}); + // Walk every frame in the protobuf + for (size_t fi = 0; fi < objMessage.frame_size(); ++fi) { + const auto &pbFrame = objMessage.frame(fi); + size_t frameId = pbFrame.id(); + + // Buffers for DetectionData + std::vector classIds; + std::vector confidences; + std::vector> boxes; + std::vector objectIds; + + // For each bounding box in this frame + for (int di = 0; di < pbFrame.bounding_box_size(); ++di) { + const auto &b = pbFrame.bounding_box(di); + float x = b.x(), y = b.y(), w = b.w(), h = b.h(); + int classId = b.classid(); + float confidence= b.confidence(); + int objectId = b.objectid(); + + // Record for DetectionData + classIds.push_back(classId); + confidences.push_back(confidence); + boxes.emplace_back(x, y, w, h); + objectIds.push_back(objectId); + + // Either append to an existing TrackedObjectBBox… + auto it = trackedObjects.find(objectId); + if (it != trackedObjects.end()) { + it->second->AddBox(frameId, x + w/2, y + h/2, w, h, 0.0); + } + else { + // …or create a brand-new one + TrackedObjectBBox tmpObj( + (int)classesColor[classId][0], + (int)classesColor[classId][1], + (int)classesColor[classId][2], + /*alpha=*/0 + ); + tmpObj.stroke_alpha = Keyframe(1.0); + tmpObj.AddBox(frameId, x + w/2, y + h/2, w, h, 0.0); + + auto ptr = std::make_shared(tmpObj); + ptr->ParentClip(this->ParentClip()); + + // Prefix with effect UUID for a unique string ID + std::string prefix = this->Id(); + if (!prefix.empty()) + prefix += "-"; + ptr->Id(prefix + std::to_string(objectId)); + trackedObjects.emplace(objectId, ptr); } - - // Create OpenCV rectangle with the bouding box info - cv::Rect_ box(x, y, w, h); - - // Push back data into vectors - boxes.push_back(box); - classIds.push_back(classId); - confidences.push_back(confidence); - objectIds.push_back(objectId); } - // Assign data to object detector map - detectionsData[id] = DetectionData(classIds, confidences, boxes, id, objectIds); - } + // Save the DetectionData for this frame + detectionsData[frameId] = DetectionData( + classIds, confidences, boxes, frameId, objectIds + ); + } - // Delete all global objects allocated by libprotobuf. - google::protobuf::ShutdownProtobufLibrary(); + google::protobuf::ShutdownProtobufLibrary(); + + // Finally, pick a default selectedObjectIndex if we have any + if (!trackedObjects.empty()) { + selectedObjectIndex = trackedObjects.begin()->first; + } - return true; + return true; } // Get the indexes and IDs of all visible objects in the given frame @@ -377,68 +356,82 @@ void ObjectDetection::SetJson(const std::string value) { } // Load Json::Value into this object -void ObjectDetection::SetJsonValue(const Json::Value root) { - // Set parent data +void ObjectDetection::SetJsonValue(const Json::Value root) +{ + // Parent properties EffectBase::SetJsonValue(root); - // Set data from Json (if key is found) - if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1){ - protobuf_data_path = root["protobuf_data_path"].asString(); - - if(!LoadObjDetectdData(protobuf_data_path)){ - throw InvalidFile("Invalid protobuf data path", ""); - protobuf_data_path = ""; + // If a protobuf path is provided, load & prefix IDs + if (!root["protobuf_data_path"].isNull()) { + std::string new_path = root["protobuf_data_path"].asString(); + if (protobuf_data_path != new_path || trackedObjects.empty()) { + protobuf_data_path = new_path; + if (!LoadObjDetectdData(protobuf_data_path)) { + throw InvalidFile("Invalid protobuf data path", ""); + } } } - // Set the selected object index + // Selected index, thresholds, UI flags, filters, etc. if (!root["selected_object_index"].isNull()) - selectedObjectIndex = root["selected_object_index"].asInt(); - + selectedObjectIndex = root["selected_object_index"].asInt(); if (!root["confidence_threshold"].isNull()) - confidence_threshold = root["confidence_threshold"].asFloat(); - + confidence_threshold = root["confidence_threshold"].asFloat(); if (!root["display_box_text"].isNull()) - display_box_text.SetJsonValue(root["display_box_text"]); - - if (!root["display_boxes"].isNull()) - display_boxes.SetJsonValue(root["display_boxes"]); - - if (!root["class_filter"].isNull()) { - class_filter = root["class_filter"].asString(); - - // Convert the class_filter to a QString - QString qClassFilter = QString::fromStdString(root["class_filter"].asString()); - - // Split the QString by commas and automatically trim each resulting string - QStringList classList = qClassFilter.split(','); - classList.removeAll(""); // Skip empty parts - display_classes.clear(); + display_box_text.SetJsonValue(root["display_box_text"]); + if (!root["display_boxes"].isNull()) + display_boxes.SetJsonValue(root["display_boxes"]); + + if (!root["class_filter"].isNull()) { + class_filter = root["class_filter"].asString(); + QStringList parts = QString::fromStdString(class_filter).split(','); + display_classes.clear(); + for (auto &p : parts) { + auto s = p.trimmed().toLower(); + if (!s.isEmpty()) { + display_classes.push_back(s.toStdString()); + } + } + } - // Iterate over the QStringList and add each trimmed, non-empty string - for (const QString &classItem : classList) { - QString trimmedItem = classItem.trimmed().toLower(); - if (!trimmedItem.isEmpty()) { - display_classes.push_back(trimmedItem.toStdString()); - } - } - } + // Apply any per-object overrides + if (!root["objects"].isNull()) { + // Iterate over the supplied objects (indexed by id or position) + const auto memberNames = root["objects"].getMemberNames(); + for (const auto& name : memberNames) + { + // Determine the numeric index of this object + int index = -1; + bool numeric_key = std::all_of(name.begin(), name.end(), ::isdigit); + if (numeric_key) { + index = std::stoi(name); + } + else + { + size_t pos = name.find_last_of('-'); + if (pos != std::string::npos) { + try { + index = std::stoi(name.substr(pos + 1)); + } catch (...) { + index = -1; + } + } + } - if (!root["objects"].isNull()){ - for (auto const& trackedObject : trackedObjects){ - std::string obj_id = std::to_string(trackedObject.first); - if(!root["objects"][obj_id].isNull()){ - trackedObject.second->SetJsonValue(root["objects"][obj_id]); + auto obj_it = trackedObjects.find(index); + if (obj_it != trackedObjects.end() && obj_it->second) { + // Update object id if provided as a non-numeric key + if (!numeric_key) + obj_it->second->Id(name); + obj_it->second->SetJsonValue(root["objects"][name]); } } } - - // Set the tracked object's ids - if (!root["objects_id"].isNull()){ - for (auto const& trackedObject : trackedObjects){ - Json::Value trackedObjectJSON; - trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString(); - trackedObject.second->SetJsonValue(trackedObjectJSON); + // Set the tracked object's ids (legacy format) + if (!root["objects_id"].isNull()) { + for (auto& kv : trackedObjects) { + if (!root["objects_id"][kv.first].isNull()) + kv.second->Id(root["objects_id"][kv.first].asString()); } } } @@ -468,9 +461,9 @@ std::string ObjectDetection::PropertiesJSON(int64_t requested_frame) const { root["display_box_text"]["choices"].append(add_property_choice_json("Yes", true, display_box_text.GetValue(requested_frame))); root["display_box_text"]["choices"].append(add_property_choice_json("No", false, display_box_text.GetValue(requested_frame))); - root["display_boxes"] = add_property_json("Draw All Boxes", display_boxes.GetValue(requested_frame), "int", "", &display_boxes, 0, 1, false, requested_frame); - root["display_boxes"]["choices"].append(add_property_choice_json("Yes", true, display_boxes.GetValue(requested_frame))); - root["display_boxes"]["choices"].append(add_property_choice_json("No", false, display_boxes.GetValue(requested_frame))); + root["display_boxes"] = add_property_json("Draw All Boxes", display_boxes.GetValue(requested_frame), "int", "", &display_boxes, 0, 1, false, requested_frame); + root["display_boxes"]["choices"].append(add_property_choice_json("Yes", true, display_boxes.GetValue(requested_frame))); + root["display_boxes"]["choices"].append(add_property_choice_json("No", false, display_boxes.GetValue(requested_frame))); // Return formatted string return root.toStyledString(); diff --git a/src/effects/ObjectDetection.h b/src/effects/ObjectDetection.h index d56eca721..1675bfca6 100644 --- a/src/effects/ObjectDetection.h +++ b/src/effects/ObjectDetection.h @@ -82,8 +82,6 @@ namespace openshot /// Index of the Tracked Object that was selected to modify it's properties int selectedObjectIndex; - ObjectDetection(std::string clipTrackerDataPath); - /// Default constructor ObjectDetection(); diff --git a/src/effects/SphericalProjection.cpp b/src/effects/SphericalProjection.cpp index 0565a7a30..bf676b272 100644 --- a/src/effects/SphericalProjection.cpp +++ b/src/effects/SphericalProjection.cpp @@ -13,250 +13,519 @@ #include "SphericalProjection.h" #include "Exceptions.h" -#include #include +#include #include +#include using namespace openshot; SphericalProjection::SphericalProjection() - : yaw(0.0) - , pitch(0.0) - , roll(0.0) - , fov(90.0) - , projection_mode(0) - , invert(0) - , interpolation(0) + : yaw(0.0), pitch(0.0), roll(0.0), fov(90.0), in_fov(180.0), + projection_mode(0), invert(0), input_model(INPUT_EQUIRECT), interpolation(3) { - init_effect_details(); + init_effect_details(); } -SphericalProjection::SphericalProjection(Keyframe new_yaw, - Keyframe new_pitch, - Keyframe new_roll, - Keyframe new_fov) - : yaw(new_yaw), pitch(new_pitch), roll(new_roll) - , fov(new_fov), projection_mode(0), invert(0), interpolation(0) +SphericalProjection::SphericalProjection(Keyframe new_yaw, Keyframe new_pitch, + Keyframe new_roll, Keyframe new_fov) + : yaw(new_yaw), pitch(new_pitch), roll(new_roll), fov(new_fov), + in_fov(180.0), projection_mode(0), invert(0), + input_model(INPUT_EQUIRECT), interpolation(3) { - init_effect_details(); + init_effect_details(); } void SphericalProjection::init_effect_details() { - InitEffectInfo(); - info.class_name = "SphericalProjection"; - info.name = "Spherical Projection"; - info.description = "Flatten and reproject 360° video with yaw, pitch, roll, and fov (sphere, hemisphere, fisheye modes)"; - info.has_audio = false; - info.has_video = true; + InitEffectInfo(); + info.class_name = "SphericalProjection"; + info.name = "Spherical Projection"; + info.description = + "Flatten and reproject 360° or fisheye inputs into a rectilinear view with yaw, pitch, roll, and FOV. Supports Equirect and multiple fisheye lens models."; + info.has_audio = false; + info.has_video = true; } -std::shared_ptr SphericalProjection::GetFrame( - std::shared_ptr frame, - int64_t frame_number) -{ - auto img = frame->GetImage(); - if (img->format() != QImage::Format_ARGB32) - *img = img->convertToFormat(QImage::Format_ARGB32); - - int W = img->width(), H = img->height(); - int bpl = img->bytesPerLine(); - uchar* src = img->bits(); - - QImage output(W, H, QImage::Format_ARGB32); - output.fill(Qt::black); - uchar* dst = output.bits(); - int dst_bpl = output.bytesPerLine(); - - // Evaluate keyframes (note roll is inverted + offset 180°) - double yaw_r = yaw.GetValue(frame_number) * M_PI/180.0; - double pitch_r = pitch.GetValue(frame_number) * M_PI/180.0; - double roll_r = -roll.GetValue(frame_number) * M_PI/180.0 + M_PI; - double fov_r = fov.GetValue(frame_number) * M_PI/180.0; - - // Build composite rotation matrix R = Ry * Rx * Rz - double sy = sin(yaw_r), cy = cos(yaw_r); - double sp = sin(pitch_r), cp = cos(pitch_r); - double sr = sin(roll_r), cr = cos(roll_r); - - double r00 = cy*cr + sy*sp*sr, r01 = -cy*sr + sy*sp*cr, r02 = sy*cp; - double r10 = cp*sr, r11 = cp*cr, r12 = -sp; - double r20 = -sy*cr + cy*sp*sr, r21 = sy*sr + cy*sp*cr, r22 = cy*cp; - - // Precompute perspective scalars - double hx = tan(fov_r*0.5); - double vy = hx * double(H)/W; +namespace { + inline double cubic_interp(double p0, double p1, double p2, double p3, + double t) + { + double a0 = -0.5 * p0 + 1.5 * p1 - 1.5 * p2 + 0.5 * p3; + double a1 = p0 - 2.5 * p1 + 2.0 * p2 - 0.5 * p3; + double a2 = -0.5 * p0 + 0.5 * p2; + double a3 = p1; + return ((a0 * t + a1) * t + a2) * t + a3; + } +} // namespace + +std::shared_ptr +SphericalProjection::GetFrame(std::shared_ptr frame, + int64_t frame_number) { + auto img = frame->GetImage(); + if (img->format() != QImage::Format_ARGB32) + *img = img->convertToFormat(QImage::Format_ARGB32); + + int W = img->width(), H = img->height(); + int bpl = img->bytesPerLine(); + uchar *src = img->bits(); + + QImage output(W, H, QImage::Format_ARGB32); + output.fill(Qt::black); + uchar *dst = output.bits(); + int dst_bpl = output.bytesPerLine(); + + // Keyframes / angles + const double DEG = M_PI / 180.0; + double yaw_r = -yaw.GetValue(frame_number) * DEG; // drag right -> look right + double pitch_r = pitch.GetValue(frame_number) * DEG; // drag up -> look up + double roll_r = -roll.GetValue(frame_number) * DEG; // positive slider -> clockwise on screen + double in_fov_r = in_fov.GetValue(frame_number) * DEG; + double out_fov_r= fov.GetValue(frame_number) * DEG; + + // Apply invert as a 180° yaw for equirect inputs (camera-centric; no mirroring) + if (input_model == INPUT_EQUIRECT && invert == INVERT_BACK) { + yaw_r += M_PI; + } + + // Rotation R = Ry(yaw) * Rx(pitch). (Roll applied in screen space.) + double sy = sin(yaw_r), cy = cos(yaw_r); + double sp = sin(pitch_r),cp = cos(pitch_r); + + double r00 = cy; + double r01 = sy * sp; + double r02 = sy * cp; + + double r10 = 0.0; + double r11 = cp; + double r12 = -sp; + + double r20 = -sy; + double r21 = cy * sp; + double r22 = cy * cp; + + // Keep roll clockwise on screen regardless of facing direction + double roll_sign = (r22 >= 0.0) ? 1.0 : -1.0; + + // Perspective scalars (rectilinear) + double hx = tan(out_fov_r * 0.5); + double vy = hx * double(H) / W; + + auto q = [](double a) { return std::llround(a * 1e6); }; + bool recompute = uv_map.empty() || W != cached_width || H != cached_height || + q(yaw_r) != q(cached_yaw) || + q(pitch_r) != q(cached_pitch) || + q(roll_r) != q(cached_roll) || + q(in_fov_r) != q(cached_in_fov) || + q(out_fov_r) != q(cached_out_fov) || + input_model != cached_input_model || + projection_mode != cached_projection_mode || + invert != cached_invert; + + if (recompute) { + uv_map.resize(W * H * 2); + +#pragma omp parallel for schedule(static) + for (int yy = 0; yy < H; yy++) { + double ndc_y = (2.0 * (yy + 0.5) / H - 1.0) * vy; + + for (int xx = 0; xx < W; xx++) { + double uf = -1.0, vf = -1.0; + + const bool out_is_rect = + (projection_mode == MODE_RECT_SPHERE || projection_mode == MODE_RECT_HEMISPHERE); + + if (!out_is_rect) { + // ---------------- FISHEYE OUTPUT ---------------- + double cx = (xx + 0.5) - W * 0.5; + double cy_dn = (yy + 0.5) - H * 0.5; + double R = 0.5 * std::min(W, H); + + // screen plane, Y-up; apply roll by -roll (clockwise), adjusted by roll_sign + double rx = cx / R; + double ry_up = -cy_dn / R; + double cR = cos(roll_r), sR = sin(roll_r) * roll_sign; + double rxr = cR * rx + sR * ry_up; + double ryr = -sR * rx + cR * ry_up; + + double r_norm = std::sqrt(rxr * rxr + ryr * ryr); + if (r_norm <= 1.0) { + double theta_max = out_fov_r * 0.5; + double theta = 0.0; + switch (projection_mode) { + case MODE_FISHEYE_EQUIDISTANT: + // r ∝ θ + theta = r_norm * theta_max; + break; + case MODE_FISHEYE_EQUISOLID: + // r ∝ 2 sin(θ/2) + theta = 2.0 * std::asin(std::clamp(r_norm * std::sin(theta_max * 0.5), -1.0, 1.0)); + break; + case MODE_FISHEYE_STEREOGRAPHIC: + // r ∝ 2 tan(θ/2) + theta = 2.0 * std::atan(r_norm * std::tan(theta_max * 0.5)); + break; + case MODE_FISHEYE_ORTHOGRAPHIC: + // r ∝ sin(θ) + theta = std::asin(std::clamp(r_norm * std::sin(theta_max), -1.0, 1.0)); + break; + default: + theta = r_norm * theta_max; + break; + } + + // NOTE: Y was upside-down; fix by using +ryr (not -ryr) + double phi = std::atan2(ryr, rxr); + + // Camera ray from fisheye output + double vx = std::sin(theta) * std::cos(phi); + double vy2= std::sin(theta) * std::sin(phi); + double vz = -std::cos(theta); + + // Rotate into world + double dx = r00 * vx + r01 * vy2 + r02 * vz; + double dy = r10 * vx + r11 * vy2 + r12 * vz; + double dz = r20 * vx + r21 * vy2 + r22 * vz; + + project_input(dx, dy, dz, in_fov_r, W, H, uf, vf); + } else { + uf = vf = -1.0; // outside disk + } + + } else { + // ---------------- RECTILINEAR OUTPUT ---------------- + double ndc_x = (2.0 * (xx + 0.5) / W - 1.0) * hx; + + // screen plane Y-up; roll by -roll (clockwise), adjusted by roll_sign + double sx = ndc_x; + double sy_up = -ndc_y; + double cR = cos(roll_r), sR = sin(roll_r) * roll_sign; + double rx = cR * sx + sR * sy_up; + double ry = -sR * sx + cR * sy_up; + + // Camera ray (camera looks down -Z) + double vx = rx, vy2 = ry, vz = -1.0; + double inv_len = 1.0 / std::sqrt(vx*vx + vy2*vy2 + vz*vz); + vx *= inv_len; vy2 *= inv_len; vz *= inv_len; + + // Rotate into world + double dx = r00 * vx + r01 * vy2 + r02 * vz; + double dy = r10 * vx + r11 * vy2 + r12 * vz; + double dz = r20 * vx + r21 * vy2 + r22 * vz; + + project_input(dx, dy, dz, in_fov_r, W, H, uf, vf); + } + + int idx = 2 * (yy * W + xx); + uv_map[idx] = (float)uf; + uv_map[idx + 1] = (float)vf; + } + } + + cached_width = W; + cached_height = H; + cached_yaw = yaw_r; + cached_pitch = pitch_r; + cached_roll = roll_r; + cached_in_fov = in_fov_r; + cached_out_fov= out_fov_r; + cached_input_model = input_model; + cached_projection_mode = projection_mode; + cached_invert = invert; + } + + // Auto sampler selection (uses enums) + int sampler = interpolation; + if (interpolation == INTERP_AUTO) { + double coverage_r = + (projection_mode == MODE_RECT_SPHERE) ? 2.0 * M_PI : + (projection_mode == MODE_RECT_HEMISPHERE) ? M_PI : + in_fov_r; // rough heuristic otherwise + double ppd_src = W / coverage_r; + double ppd_out = W / out_fov_r; + double ratio = ppd_out / ppd_src; + if (ratio < 0.8) sampler = INTERP_AUTO; // mipmaps path below + else if (ratio <= 1.2) sampler = INTERP_BILINEAR; + else sampler = INTERP_BICUBIC; + } + + // Build mipmaps only if needed (box) + std::vector mipmaps; + if (sampler == INTERP_AUTO) { + mipmaps.push_back(*img); + for (int level = 1; level < 4; ++level) { + const QImage &prev = mipmaps[level - 1]; + if (prev.width() <= 1 || prev.height() <= 1) break; + int w = prev.width() / 2, h = prev.height() / 2; + QImage next(w, h, QImage::Format_ARGB32); + uchar *nb = next.bits(); int nbpl = next.bytesPerLine(); + const uchar *pb = prev.bits(); int pbpl = prev.bytesPerLine(); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + for (int c = 0; c < 4; c++) { + int p00 = pb[(2*y) * pbpl + (2*x) * 4 + c]; + int p10 = pb[(2*y) * pbpl + (2*x+1) * 4 + c]; + int p01 = pb[(2*y+1) * pbpl + (2*x) * 4 + c]; + int p11 = pb[(2*y+1) * pbpl + (2*x+1) * 4 + c]; + nb[y * nbpl + x * 4 + c] = (p00 + p10 + p01 + p11) / 4; + } + } + } + mipmaps.push_back(next); + } + } #pragma omp parallel for schedule(static) - for (int yy = 0; yy < H; yy++) { - uchar* dst_row = dst + yy * dst_bpl; - double ndc_y = (2.0*(yy + 0.5)/H - 1.0) * vy; - - for (int xx = 0; xx < W; xx++) { - // Generate ray in camera space - double ndc_x = (2.0*(xx + 0.5)/W - 1.0) * hx; - double vx = ndc_x, vy2 = -ndc_y, vz = -1.0; - double inv = 1.0/sqrt(vx*vx + vy2*vy2 + vz*vz); - vx *= inv; vy2 *= inv; vz *= inv; - - // Rotate ray into world coordinates - double dx = r00*vx + r01*vy2 + r02*vz; - double dy = r10*vx + r11*vy2 + r12*vz; - double dz = r20*vx + r21*vy2 + r22*vz; - - // For sphere/hemisphere, optionally invert view by 180° - if (projection_mode < 2 && invert) { - dx = -dx; - dz = -dz; - } - - double uf, vf; - - if (projection_mode == 2) { - // Fisheye mode: invert circular fisheye - double ax = 0.0, ay = 0.0, az = invert ? -1.0 : 1.0; - double cos_t = dx*ax + dy*ay + dz*az; - double theta = acos(cos_t); - double rpx = (theta / fov_r) * (W/2.0); - double phi = atan2(dy, dx); - uf = W*0.5 + rpx*cos(phi); - vf = H*0.5 + rpx*sin(phi); - } - else { - // Sphere or hemisphere: equirectangular sampling - double lon = atan2(dx, dz); - double lat = asin(dy); - if (projection_mode == 1) // hemisphere - lon = std::clamp(lon, -M_PI/2.0, M_PI/2.0); - uf = ((lon + (projection_mode? M_PI/2.0 : M_PI)) - / (projection_mode? M_PI : 2.0*M_PI)) * W; - vf = (lat + M_PI/2.0)/M_PI * H; - } - - uchar* d = dst_row + xx*4; - - if (interpolation == 0) { - // Nearest-neighbor sampling - int x0 = std::clamp(int(std::floor(uf)), 0, W-1); - int y0 = std::clamp(int(std::floor(vf)), 0, H-1); - uchar* s = src + y0*bpl + x0*4; - d[0] = s[0]; d[1] = s[1]; d[2] = s[2]; d[3] = s[3]; - } - else { - // Bilinear sampling - int x0 = std::clamp(int(std::floor(uf)), 0, W-1); - int y0 = std::clamp(int(std::floor(vf)), 0, H-1); - int x1 = std::clamp(x0 + 1, 0, W-1); - int y1 = std::clamp(y0 + 1, 0, H-1); - double dxr = uf - x0, dyr = vf - y0; - uchar* p00 = src + y0*bpl + x0*4; - uchar* p10 = src + y0*bpl + x1*4; - uchar* p01 = src + y1*bpl + x0*4; - uchar* p11 = src + y1*bpl + x1*4; - for (int c = 0; c < 4; c++) { - double v0 = p00[c]*(1-dxr) + p10[c]*dxr; - double v1 = p01[c]*(1-dxr) + p11[c]*dxr; - d[c] = uchar(v0*(1-dyr) + v1*dyr + 0.5); - } - } - } - } - - *img = output; - return frame; + for (int yy = 0; yy < H; yy++) { + uchar *dst_row = dst + yy * dst_bpl; + for (int xx = 0; xx < W; xx++) { + int idx = 2 * (yy * W + xx); + double uf = uv_map[idx]; + double vf = uv_map[idx + 1]; + uchar *d = dst_row + xx * 4; + + if (input_model == INPUT_EQUIRECT && projection_mode == MODE_RECT_SPHERE) { + uf = std::fmod(std::fmod(uf, W) + W, W); + vf = std::clamp(vf, 0.0, (double)H - 1); + } else if (input_model == INPUT_EQUIRECT && projection_mode == MODE_RECT_HEMISPHERE) { + uf = std::clamp(uf, 0.0, (double)W - 1); + vf = std::clamp(vf, 0.0, (double)H - 1); + } else if (uf < 0 || uf >= W || vf < 0 || vf >= H) { + d[0] = d[1] = d[2] = 0; d[3] = 0; + continue; + } + + if (sampler == INTERP_NEAREST) { + int x0 = std::clamp(int(std::floor(uf)), 0, W - 1); + int y0 = std::clamp(int(std::floor(vf)), 0, H - 1); + uchar *s = src + y0 * bpl + x0 * 4; + d[0]=s[0]; d[1]=s[1]; d[2]=s[2]; d[3]=s[3]; + } else if (sampler == INTERP_BILINEAR) { + int x0 = std::clamp(int(std::floor(uf)), 0, W - 1); + int y0 = std::clamp(int(std::floor(vf)), 0, H - 1); + int x1 = std::clamp(x0 + 1, 0, W - 1); + int y1 = std::clamp(y0 + 1, 0, H - 1); + double dxr = uf - x0, dyr = vf - y0; + uchar *p00 = src + y0 * bpl + x0 * 4; + uchar *p10 = src + y0 * bpl + x1 * 4; + uchar *p01 = src + y1 * bpl + x0 * 4; + uchar *p11 = src + y1 * bpl + x1 * 4; + for (int c = 0; c < 4; c++) { + double v0 = p00[c] * (1 - dxr) + p10[c] * dxr; + double v1 = p01[c] * (1 - dxr) + p11[c] * dxr; + d[c] = uchar(v0 * (1 - dyr) + v1 * dyr + 0.5); + } + } else if (sampler == INTERP_BICUBIC) { + int x1 = std::clamp(int(std::floor(uf)), 0, W - 1); + int y1 = std::clamp(int(std::floor(vf)), 0, H - 1); + double tx = uf - x1, ty = vf - y1; + for (int c = 0; c < 4; c++) { + double col[4]; + for (int j = -1; j <= 2; j++) { + int y = std::clamp(y1 + j, 0, H - 1); + double row[4]; + for (int i = -1; i <= 2; i++) { + int x = std::clamp(x1 + i, 0, W - 1); + row[i + 1] = src[y * bpl + x * 4 + c]; + } + col[j + 1] = cubic_interp(row[0], row[1], row[2], row[3], tx); + } + double val = cubic_interp(col[0], col[1], col[2], col[3], ty); + d[c] = uchar(std::clamp(val, 0.0, 255.0) + 0.5); + } + } else { // INTERP_AUTO -> mipmaps + bilinear + double uf_dx = 0.0, vf_dx = 0.0, uf_dy = 0.0, vf_dy = 0.0; + if (xx + 1 < W) { uf_dx = uv_map[idx + 2] - uf; vf_dx = uv_map[idx + 3] - vf; } + if (yy + 1 < H) { uf_dy = uv_map[idx + 2 * W] - uf; vf_dy = uv_map[idx + 2 * W + 1] - vf; } + double scale_x = std::sqrt(uf_dx*uf_dx + vf_dx*vf_dx); + double scale_y = std::sqrt(uf_dy*uf_dy + vf_dy*vf_dy); + double scale = std::max(scale_x, scale_y); + int level = 0; + if (scale > 1.0) + level = std::min(std::floor(std::log2(scale)), (int)mipmaps.size() - 1); + const QImage &lvl = mipmaps[level]; + int Wl = lvl.width(), Hl = lvl.height(); + int bpl_l = lvl.bytesPerLine(); + const uchar *srcl = lvl.bits(); + double uf_l = uf / (1 << level); + double vf_l = vf / (1 << level); + int x0 = std::clamp(int(std::floor(uf_l)), 0, Wl - 1); + int y0 = std::clamp(int(std::floor(vf_l)), 0, Hl - 1); + int x1 = std::clamp(x0 + 1, 0, Wl - 1); + int y1 = std::clamp(y0 + 1, 0, Hl - 1); + double dxr = uf_l - x0, dyr = vf_l - y0; + const uchar *p00 = srcl + y0 * bpl_l + x0 * 4; + const uchar *p10 = srcl + y0 * bpl_l + x1 * 4; + const uchar *p01 = srcl + y1 * bpl_l + x0 * 4; + const uchar *p11 = srcl + y1 * bpl_l + x1 * 4; + for (int c = 0; c < 4; c++) { + double v0 = p00[c] * (1 - dxr) + p10[c] * dxr; + double v1 = p01[c] * (1 - dxr) + p11[c] * dxr; + d[c] = uchar(v0 * (1 - dyr) + v1 * dyr + 0.5); + } + } + } + } + + *img = output; + return frame; +} + +void SphericalProjection::project_input(double dx, double dy, double dz, + double in_fov_r, int W, int H, + double &uf, double &vf) const { + if (input_model == INPUT_EQUIRECT) { + // Center (-Z) -> lon=0; +X (screen right) -> +lon + double lon = std::atan2(dx, -dz); + double lat = std::asin(std::clamp(dy, -1.0, 1.0)); + + if (projection_mode == MODE_RECT_HEMISPHERE) + lon = std::clamp(lon, -M_PI / 2.0, M_PI / 2.0); + + double horiz_span = (projection_mode == MODE_RECT_HEMISPHERE) ? M_PI : 2.0 * M_PI; + double lon_offset = (projection_mode == MODE_RECT_HEMISPHERE) ? M_PI / 2.0 : M_PI; + uf = ((lon + lon_offset) / horiz_span) * W; + + // Image Y grows downward: north (lat = +π/2) at top + vf = (M_PI / 2.0 - lat) / M_PI * H; + return; + } + + // -------- Fisheye inputs -------- + // Optical axis default is -Z; "Invert" flips hemisphere. + const double ax = 0.0, ay = 0.0; + double az = -1.0; + if (invert == INVERT_BACK) az = 1.0; + + double cos_t = std::clamp(dx * ax + dy * ay + dz * az, -1.0, 1.0); + double theta = std::acos(cos_t); + double tmax = std::max(1e-6, in_fov_r * 0.5); + + double r_norm = 0.0; + switch (input_model) { + case INPUT_FEQ_EQUIDISTANT: r_norm = theta / tmax; break; + case INPUT_FEQ_EQUISOLID: r_norm = std::sin(theta*0.5) / std::max(1e-12, std::sin(tmax*0.5)); break; + case INPUT_FEQ_STEREOGRAPHIC: r_norm = std::tan(theta*0.5) / std::max(1e-12, std::tan(tmax*0.5)); break; + case INPUT_FEQ_ORTHOGRAPHIC: r_norm = std::sin(theta) / std::max(1e-12, std::sin(tmax)); break; + default: r_norm = theta / tmax; break; + } + + // Azimuth in camera XY; final Y is downward -> subtract sine in vf + double phi = std::atan2(dy, dx); + + double R = 0.5 * std::min(W, H); + double rpx = r_norm * R; + uf = W * 0.5 + rpx * std::cos(phi); + vf = H * 0.5 - rpx * std::sin(phi); } std::string SphericalProjection::Json() const { - return JsonValue().toStyledString(); + return JsonValue().toStyledString(); } Json::Value SphericalProjection::JsonValue() const { - Json::Value root = EffectBase::JsonValue(); - root["type"] = info.class_name; - root["yaw"] = yaw.JsonValue(); - root["pitch"] = pitch.JsonValue(); - root["roll"] = roll.JsonValue(); - root["fov"] = fov.JsonValue(); - root["projection_mode"] = projection_mode; - root["invert"] = invert; - root["interpolation"] = interpolation; - return root; + Json::Value root = EffectBase::JsonValue(); + root["type"] = info.class_name; + root["yaw"] = yaw.JsonValue(); + root["pitch"] = pitch.JsonValue(); + root["roll"] = roll.JsonValue(); + root["fov"] = fov.JsonValue(); + root["in_fov"] = in_fov.JsonValue(); + root["projection_mode"] = projection_mode; + root["invert"] = invert; + root["input_model"] = input_model; + root["interpolation"] = interpolation; + return root; } void SphericalProjection::SetJson(const std::string value) { - try { - Json::Value root = openshot::stringToJson(value); - SetJsonValue(root); - } - catch (...) { - throw InvalidJSON("Invalid JSON for SphericalProjection"); - } + try + { + Json::Value root = openshot::stringToJson(value); + SetJsonValue(root); + } + catch (...) + { + throw InvalidJSON("Invalid JSON for SphericalProjection"); + } } void SphericalProjection::SetJsonValue(const Json::Value root) { - EffectBase::SetJsonValue(root); - if (!root["yaw"].isNull()) yaw.SetJsonValue(root["yaw"]); - if (!root["pitch"].isNull()) pitch.SetJsonValue(root["pitch"]); - if (!root["roll"].isNull()) roll.SetJsonValue(root["roll"]); - if (!root["fov"].isNull()) fov.SetJsonValue(root["fov"]); - if (!root["projection_mode"].isNull()) projection_mode = root["projection_mode"].asInt(); - if (!root["invert"].isNull()) invert = root["invert"].asInt(); - if (!root["interpolation"].isNull()) interpolation = root["interpolation"].asInt(); -} + EffectBase::SetJsonValue(root); + + if (!root["yaw"].isNull()) yaw.SetJsonValue(root["yaw"]); + if (!root["pitch"].isNull()) pitch.SetJsonValue(root["pitch"]); + if (!root["roll"].isNull()) roll.SetJsonValue(root["roll"]); + if (!root["fov"].isNull()) fov.SetJsonValue(root["fov"]); + if (!root["in_fov"].isNull()) in_fov.SetJsonValue(root["in_fov"]); + if (!root["projection_mode"].isNull()) + projection_mode = root["projection_mode"].asInt(); + + if (!root["invert"].isNull()) + invert = root["invert"].asInt(); + + if (!root["input_model"].isNull()) + input_model = root["input_model"].asInt(); + + if (!root["interpolation"].isNull()) + interpolation = root["interpolation"].asInt(); + + // Clamp to enum options + projection_mode = std::clamp(projection_mode, + (int)MODE_RECT_SPHERE, + (int)MODE_FISHEYE_ORTHOGRAPHIC); + invert = std::clamp(invert, (int)INVERT_NORMAL, (int)INVERT_BACK); + input_model = std::clamp(input_model, (int)INPUT_EQUIRECT, (int)INPUT_FEQ_ORTHOGRAPHIC); + interpolation = std::clamp(interpolation, (int)INTERP_NEAREST, (int)INTERP_AUTO); + + // any property change should invalidate cached UV map + uv_map.clear(); + + + // any property change should invalidate cached UV map + uv_map.clear(); +} std::string SphericalProjection::PropertiesJSON(int64_t requested_frame) const { - Json::Value root = BasePropertiesJSON(requested_frame); - - root["yaw"] = add_property_json("Yaw", - yaw.GetValue(requested_frame), - "float", "degrees", - &yaw, -180, 180, - false, requested_frame); - root["pitch"] = add_property_json("Pitch", - pitch.GetValue(requested_frame), - "float", "degrees", - &pitch, -90, 90, - false, requested_frame); - root["roll"] = add_property_json("Roll", - roll.GetValue(requested_frame), - "float", "degrees", - &roll, -180, 180, - false, requested_frame); - root["fov"] = add_property_json("FOV", - fov.GetValue(requested_frame), - "float", "degrees", - &fov, 1, 179, - false, requested_frame); - - root["projection_mode"] = add_property_json("Projection Mode", - projection_mode, - "int", "", - nullptr, 0, 2, - false, requested_frame); - root["projection_mode"]["choices"].append(add_property_choice_json("Sphere", 0, projection_mode)); - root["projection_mode"]["choices"].append(add_property_choice_json("Hemisphere", 1, projection_mode)); - root["projection_mode"]["choices"].append(add_property_choice_json("Fisheye", 2, projection_mode)); - - root["invert"] = add_property_json("Invert View", - invert, - "int", "", - nullptr, 0, 1, - false, requested_frame); - root["invert"]["choices"].append(add_property_choice_json("Normal", 0, invert)); - root["invert"]["choices"].append(add_property_choice_json("Invert", 1, invert)); - - root["interpolation"] = add_property_json("Interpolation", - interpolation, - "int", "", - nullptr, 0, 1, - false, requested_frame); - root["interpolation"]["choices"].append(add_property_choice_json("Nearest", 0, interpolation)); - root["interpolation"]["choices"].append(add_property_choice_json("Bilinear", 1, interpolation)); - - return root.toStyledString(); + Json::Value root = BasePropertiesJSON(requested_frame); + + root["yaw"] = add_property_json("Yaw", yaw.GetValue(requested_frame), "float", "degrees", &yaw, -180, 180, false, requested_frame); + root["pitch"] = add_property_json("Pitch", pitch.GetValue(requested_frame), "float", "degrees", &pitch,-180, 180, false, requested_frame); + root["roll"] = add_property_json("Roll", roll.GetValue(requested_frame), "float", "degrees", &roll, -180, 180, false, requested_frame); + + root["fov"] = add_property_json("Out FOV", fov.GetValue(requested_frame), "float", "degrees", &fov, 0, 179, false, requested_frame); + root["in_fov"] = add_property_json("In FOV", in_fov.GetValue(requested_frame), "float", "degrees", &in_fov, 1, 360, false, requested_frame); + + root["projection_mode"] = add_property_json("Projection Mode", projection_mode, "int", "", nullptr, + (int)MODE_RECT_SPHERE, (int)MODE_FISHEYE_ORTHOGRAPHIC, false, requested_frame); + root["projection_mode"]["choices"].append(add_property_choice_json("Sphere", (int)MODE_RECT_SPHERE, projection_mode)); + root["projection_mode"]["choices"].append(add_property_choice_json("Hemisphere", (int)MODE_RECT_HEMISPHERE, projection_mode)); + root["projection_mode"]["choices"].append(add_property_choice_json("Fisheye: Equidistant", (int)MODE_FISHEYE_EQUIDISTANT, projection_mode)); + root["projection_mode"]["choices"].append(add_property_choice_json("Fisheye: Equisolid", (int)MODE_FISHEYE_EQUISOLID, projection_mode)); + root["projection_mode"]["choices"].append(add_property_choice_json("Fisheye: Stereographic", (int)MODE_FISHEYE_STEREOGRAPHIC,projection_mode)); + root["projection_mode"]["choices"].append(add_property_choice_json("Fisheye: Orthographic", (int)MODE_FISHEYE_ORTHOGRAPHIC, projection_mode)); + + root["invert"] = add_property_json("Invert View", invert, "int", "", nullptr, 0, 1, false, requested_frame); + root["invert"]["choices"].append(add_property_choice_json("Normal", 0, invert)); + root["invert"]["choices"].append(add_property_choice_json("Invert", 1, invert)); + + root["input_model"] = add_property_json("Input Model", input_model, "int", "", nullptr, INPUT_EQUIRECT, INPUT_FEQ_ORTHOGRAPHIC, false, requested_frame); + root["input_model"]["choices"].append(add_property_choice_json("Equirectangular (Panorama)", INPUT_EQUIRECT, input_model)); + root["input_model"]["choices"].append(add_property_choice_json("Fisheye: Equidistant", INPUT_FEQ_EQUIDISTANT, input_model)); + root["input_model"]["choices"].append(add_property_choice_json("Fisheye: Equisolid", INPUT_FEQ_EQUISOLID, input_model)); + root["input_model"]["choices"].append(add_property_choice_json("Fisheye: Stereographic", INPUT_FEQ_STEREOGRAPHIC, input_model)); + root["input_model"]["choices"].append(add_property_choice_json("Fisheye: Orthographic", INPUT_FEQ_ORTHOGRAPHIC, input_model)); + + root["interpolation"] = add_property_json("Interpolation", interpolation, "int", "", nullptr, 0, 3, false, requested_frame); + root["interpolation"]["choices"].append(add_property_choice_json("Nearest", 0, interpolation)); + root["interpolation"]["choices"].append(add_property_choice_json("Bilinear", 1, interpolation)); + root["interpolation"]["choices"].append(add_property_choice_json("Bicubic", 2, interpolation)); + root["interpolation"]["choices"].append(add_property_choice_json("Auto", 3, interpolation)); + + return root.toStyledString(); } diff --git a/src/effects/SphericalProjection.h b/src/effects/SphericalProjection.h index 1738f976e..c8f77e274 100644 --- a/src/effects/SphericalProjection.h +++ b/src/effects/SphericalProjection.h @@ -20,53 +20,96 @@ #include #include +#include -namespace openshot -{ +namespace openshot { /** * @brief Projects 360° or fisheye video through a virtual camera. - * Supports yaw, pitch, roll, FOV, sphere/hemisphere/fisheye modes, - * optional inversion, and nearest/bilinear sampling. + * Supports yaw, pitch, roll, input and output FOV, sphere/hemisphere/fisheye + * modes, optional inversion, and automatic quality selection. */ -class SphericalProjection : public EffectBase -{ +class SphericalProjection : public EffectBase { private: - void init_effect_details(); + void init_effect_details(); public: - Keyframe yaw; ///< Yaw around up-axis (degrees) - Keyframe pitch; ///< Pitch around right-axis (degrees) - Keyframe roll; ///< Roll around forward-axis (degrees) - Keyframe fov; ///< Field-of-view (horizontal, degrees) - - int projection_mode; ///< 0=Sphere, 1=Hemisphere, 2=Fisheye - int invert; ///< 0=Normal, 1=Invert (back lens / +180°) - int interpolation; ///< 0=Nearest, 1=Bilinear - - /// Blank ctor (for JSON deserialization) - SphericalProjection(); - - /// Ctor with custom curves - SphericalProjection(Keyframe new_yaw, - Keyframe new_pitch, - Keyframe new_roll, - Keyframe new_fov); - - /// ClipBase override: create a fresh Frame then call the main GetFrame - std::shared_ptr GetFrame(int64_t frame_number) override - { return GetFrame(std::make_shared(), frame_number); } - - /// EffectBase override: reproject the QImage - std::shared_ptr GetFrame(std::shared_ptr frame, - int64_t frame_number) override; - - // JSON serialization - std::string Json() const override; - void SetJson(std::string value) override; - Json::Value JsonValue() const override; - void SetJsonValue(Json::Value root) override; - std::string PropertiesJSON(int64_t requested_frame) const override; + // Enums + enum InputModel { + INPUT_EQUIRECT = 0, + INPUT_FEQ_EQUIDISTANT = 1, // r = f * theta + INPUT_FEQ_EQUISOLID = 2, // r = 2f * sin(theta/2) + INPUT_FEQ_STEREOGRAPHIC = 3, // r = 2f * tan(theta/2) + INPUT_FEQ_ORTHOGRAPHIC = 4 // r = f * sin(theta) + }; + + enum ProjectionMode { + MODE_RECT_SPHERE = 0, // Rectilinear view over full sphere + MODE_RECT_HEMISPHERE = 1, // Rectilinear view over hemisphere + MODE_FISHEYE_EQUIDISTANT = 2, // Output fisheye (equidistant) + MODE_FISHEYE_EQUISOLID = 3, // Output fisheye (equisolid) + MODE_FISHEYE_STEREOGRAPHIC = 4, // Output fisheye (stereographic) + MODE_FISHEYE_ORTHOGRAPHIC = 5 // Output fisheye (orthographic) + }; + + enum InterpMode { + INTERP_NEAREST = 0, + INTERP_BILINEAR = 1, + INTERP_BICUBIC = 2, + INTERP_AUTO = 3 + }; + + enum InvertFlag { + INVERT_NORMAL = 0, + INVERT_BACK = 1 + }; + + Keyframe yaw; ///< Yaw around up-axis (degrees) + Keyframe pitch; ///< Pitch around right-axis (degrees) + Keyframe roll; ///< Roll around forward-axis (degrees) + Keyframe fov; ///< Output field-of-view (degrees) + Keyframe in_fov; ///< Source lens coverage / FOV (degrees) + + int projection_mode; ///< 0=Sphere, 1=Hemisphere, 2=Fisheye + int invert; ///< 0=Normal, 1=Invert (back lens / +180°) + int input_model; ///< 0=Equirect, 1=Fisheye-Equidistant + int interpolation; ///< 0=Nearest, 1=Bilinear, 2=Bicubic, 3=Auto + + /// Blank ctor (for JSON deserialization) + SphericalProjection(); + + /// Ctor with custom curves + SphericalProjection(Keyframe new_yaw, Keyframe new_pitch, Keyframe new_roll, + Keyframe new_fov); + + /// ClipBase override: create a fresh Frame then call the main GetFrame + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); + } + + /// EffectBase override: reproject the QImage + std::shared_ptr GetFrame(std::shared_ptr frame, + int64_t frame_number) override; + + // JSON serialization + std::string Json() const override; + void SetJson(std::string value) override; + Json::Value JsonValue() const override; + void SetJsonValue(Json::Value root) override; + std::string PropertiesJSON(int64_t requested_frame) const override; + +private: + void project_input(double dx, double dy, double dz, double in_fov_r, int W, + int H, double &uf, double &vf) const; + + mutable std::vector uv_map; ///< Cached UV lookup + mutable int cached_width = 0; + mutable int cached_height = 0; + mutable double cached_yaw = 0.0, cached_pitch = 0.0, cached_roll = 0.0; + mutable double cached_in_fov = 0.0, cached_out_fov = 0.0; + mutable int cached_input_model = -1; + mutable int cached_projection_mode = -1; + mutable int cached_invert = -1; }; } // namespace openshot diff --git a/src/effects/Tracker.cpp b/src/effects/Tracker.cpp index c4e023e82..2776ab7ad 100644 --- a/src/effects/Tracker.cpp +++ b/src/effects/Tracker.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "effects/Tracker.h" #include "Exceptions.h" @@ -32,38 +33,25 @@ using namespace std; using namespace openshot; using google::protobuf::util::TimeUtil; -/// Blank constructor, useful when using Json to load the effect properties -Tracker::Tracker(std::string clipTrackerDataPath) -{ - // Init effect properties - init_effect_details(); - // Instantiate a TrackedObjectBBox object and point to it - TrackedObjectBBox trackedDataObject; - trackedData = std::make_shared(trackedDataObject); - // Tries to load the tracked object's data from protobuf file - trackedData->LoadBoxData(clipTrackerDataPath); - ClipBase* parentClip = this->ParentClip(); - trackedData->ParentClip(parentClip); - trackedData->Id(std::to_string(0)); - // Insert TrackedObject with index 0 to the trackedObjects map - trackedObjects.insert({0, trackedData}); -} // Default constructor Tracker::Tracker() { - // Init effect properties + // Initialize effect metadata init_effect_details(); - // Instantiate a TrackedObjectBBox object and point to it - TrackedObjectBBox trackedDataObject; - trackedData = std::make_shared(trackedDataObject); - ClipBase* parentClip = this->ParentClip(); - trackedData->ParentClip(parentClip); - trackedData->Id(std::to_string(0)); - // Insert TrackedObject with index 0 to the trackedObjects map - trackedObjects.insert({0, trackedData}); -} + // Create a placeholder object so we always have index 0 available + trackedData = std::make_shared(); + trackedData->ParentClip(this->ParentClip()); + + // Seed our map with a single entry at index 0 + trackedObjects.clear(); + trackedObjects.emplace(0, trackedData); + + // Assign ID to the placeholder object + if (trackedData) + trackedData->Id(Id() + "-0"); +} // Init effect settings void Tracker::init_effect_details() @@ -84,73 +72,80 @@ void Tracker::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object -std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the frame's QImage - std::shared_ptr frame_image = frame->GetImage(); - - // Check if frame isn't NULL - if(frame_image && !frame_image->isNull() && - trackedData->Contains(frame_number) && - trackedData->visible.GetValue(frame_number) == 1) { - QPainter painter(frame_image.get()); - - // Get the bounding-box of the given frame - BBox fd = trackedData->GetBox(frame_number); - - // Create a QRectF for the bounding box - QRectF boxRect((fd.cx - fd.width / 2) * frame_image->width(), - (fd.cy - fd.height / 2) * frame_image->height(), - fd.width * frame_image->width(), - fd.height * frame_image->height()); - - // Check if track data exists for the requested frame - if (trackedData->draw_box.GetValue(frame_number) == 1) { - painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); - - // Get trackedObjectBox keyframes - std::vector stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number); - int stroke_width = trackedData->stroke_width.GetValue(frame_number); - float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number); - std::vector bg_rgba = trackedData->background.GetColorRGBA(frame_number); - float bg_alpha = trackedData->background_alpha.GetValue(frame_number); - float bg_corner = trackedData->background_corner.GetValue(frame_number); - - // Set the pen for the border - QPen pen(QColor(stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], 255 * stroke_alpha)); - pen.setWidth(stroke_width); - painter.setPen(pen); - - // Set the brush for the background - QBrush brush(QColor(bg_rgba[0], bg_rgba[1], bg_rgba[2], 255 * bg_alpha)); - painter.setBrush(brush); - - // Draw the rounded rectangle - painter.drawRoundedRect(boxRect, bg_corner, bg_corner); - } - - painter.end(); - } - - // No need to set the image back to the frame, as we directly modified the frame's QImage - return frame; +std::shared_ptr Tracker::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + // Sanity‐check + if (!frame) return frame; + auto frame_image = frame->GetImage(); + if (!frame_image || frame_image->isNull()) return frame; + if (!trackedData) return frame; + + // 2) Only proceed if we actually have a box and it's visible + if (!trackedData->Contains(frame_number) || + trackedData->visible.GetValue(frame_number) != 1) + return frame; + + QPainter painter(frame_image.get()); + painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform); + + // Draw the box + BBox fd = trackedData->GetBox(frame_number); + QRectF boxRect( + (fd.cx - fd.width/2) * frame_image->width(), + (fd.cy - fd.height/2) * frame_image->height(), + fd.width * frame_image->width(), + fd.height * frame_image->height() + ); + + if (trackedData->draw_box.GetValue(frame_number) == 1) + { + auto stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number); + int stroke_width = trackedData->stroke_width.GetValue(frame_number); + float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number); + auto bg_rgba = trackedData->background.GetColorRGBA(frame_number); + float bg_alpha = trackedData->background_alpha.GetValue(frame_number); + float bg_corner = trackedData->background_corner.GetValue(frame_number); + + QPen pen(QColor( + stroke_rgba[0], stroke_rgba[1], stroke_rgba[2], + int(255 * stroke_alpha) + )); + pen.setWidth(stroke_width); + painter.setPen(pen); + + QBrush brush(QColor( + bg_rgba[0], bg_rgba[1], bg_rgba[2], + int(255 * bg_alpha) + )); + painter.setBrush(brush); + + painter.drawRoundedRect(boxRect, bg_corner, bg_corner); + } + + painter.end(); + return frame; } // Get the indexes and IDs of all visible objects in the given frame -std::string Tracker::GetVisibleObjects(int64_t frame_number) const{ - - // Initialize the JSON objects +std::string Tracker::GetVisibleObjects(int64_t frame_number) const +{ Json::Value root; root["visible_objects_index"] = Json::Value(Json::arrayValue); - root["visible_objects_id"] = Json::Value(Json::arrayValue); - - // Iterate through the tracked objects - for (const auto& trackedObject : trackedObjects){ - // Get the tracked object JSON properties for this frame - Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number); - if (trackedObjectJSON["visible"]["value"].asBool()){ - // Save the object's index and ID if it's visible in this frame - root["visible_objects_index"].append(trackedObject.first); - root["visible_objects_id"].append(trackedObject.second->Id()); + root["visible_objects_id"] = Json::Value(Json::arrayValue); + + if (trackedObjects.empty()) + return root.toStyledString(); + + for (auto const& kv : trackedObjects) { + auto ptr = kv.second; + if (!ptr) continue; + + // Directly get the Json::Value for this object's properties + Json::Value propsJson = ptr->PropertiesJSON(frame_number); + + if (propsJson["visible"]["value"].asBool()) { + root["visible_objects_index"].append(kv.first); + root["visible_objects_id"].append(ptr->Id()); } } @@ -214,51 +209,82 @@ void Tracker::SetJsonValue(const Json::Value root) { // Set parent data EffectBase::SetJsonValue(root); - if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject()) - { + if (!root["BaseFPS"].isNull()) { if (!root["BaseFPS"]["num"].isNull()) - { - BaseFPS.num = (int) root["BaseFPS"]["num"].asInt(); - } + BaseFPS.num = root["BaseFPS"]["num"].asInt(); if (!root["BaseFPS"]["den"].isNull()) - { - BaseFPS.den = (int) root["BaseFPS"]["den"].asInt(); - } + BaseFPS.den = root["BaseFPS"]["den"].asInt(); } - if (!root["TimeScale"].isNull()) - TimeScale = (double) root["TimeScale"].asDouble(); + if (!root["TimeScale"].isNull()) { + TimeScale = root["TimeScale"].asDouble(); + } - // Set data from Json (if key is found) - if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1) - { - protobuf_data_path = root["protobuf_data_path"].asString(); - if(!trackedData->LoadBoxData(protobuf_data_path)) - { - std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n'; - protobuf_data_path = ""; + if (!root["protobuf_data_path"].isNull()) { + std::string new_path = root["protobuf_data_path"].asString(); + if (protobuf_data_path != new_path || trackedData->GetLength() == 0) { + protobuf_data_path = new_path; + if (!trackedData->LoadBoxData(protobuf_data_path)) { + std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n'; + protobuf_data_path.clear(); + } + else { + // prefix "-" for each entry + for (auto& kv : trackedObjects) { + auto idx = kv.first; + auto ptr = kv.second; + if (ptr) { + std::string prefix = this->Id(); + if (!prefix.empty()) + prefix += "-"; + ptr->Id(prefix + std::to_string(idx)); + } + } + } } } - if (!root["objects"].isNull()){ - for (auto const& trackedObject : trackedObjects){ - std::string obj_id = std::to_string(trackedObject.first); - if(!root["objects"][obj_id].isNull()){ - trackedObject.second->SetJsonValue(root["objects"][obj_id]); + // then any per-object JSON overrides... + if (!root["objects"].isNull()) { + // Iterate over the supplied objects (indexed by id or position) + const auto memberNames = root["objects"].getMemberNames(); + for (const auto& name : memberNames) + { + // Determine the numeric index of this object + int index = -1; + bool numeric_key = std::all_of(name.begin(), name.end(), ::isdigit); + if (numeric_key) { + index = std::stoi(name); + } + else + { + size_t pos = name.find_last_of('-'); + if (pos != std::string::npos) { + try { + index = std::stoi(name.substr(pos + 1)); + } catch (...) { + index = -1; + } + } + } + + auto obj_it = trackedObjects.find(index); + if (obj_it != trackedObjects.end() && obj_it->second) { + // Update object id if provided as a non-numeric key + if (!numeric_key) + obj_it->second->Id(name); + obj_it->second->SetJsonValue(root["objects"][name]); } } } - // Set the tracked object's ids - if (!root["objects_id"].isNull()){ - for (auto const& trackedObject : trackedObjects){ - Json::Value trackedObjectJSON; - trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString(); - trackedObject.second->SetJsonValue(trackedObjectJSON); + // Set the tracked object's ids (legacy format) + if (!root["objects_id"].isNull()) { + for (auto& kv : trackedObjects) { + if (!root["objects_id"][kv.first].isNull()) + kv.second->Id(root["objects_id"][kv.first].asString()); } } - - return; } // Get all properties for a specific frame diff --git a/src/effects/Tracker.h b/src/effects/Tracker.h index d05b72a1f..b34c376f6 100644 --- a/src/effects/Tracker.h +++ b/src/effects/Tracker.h @@ -54,8 +54,6 @@ namespace openshot /// Default constructor Tracker(); - Tracker(std::string clipTrackerDataPath); - /// @brief Apply this effect to an openshot::Frame /// /// @returns The modified openshot::Frame object diff --git a/src/effects/Wave.cpp b/src/effects/Wave.cpp index 286cb6322..9e2854eaa 100644 --- a/src/effects/Wave.cpp +++ b/src/effects/Wave.cpp @@ -51,9 +51,10 @@ std::shared_ptr Wave::GetFrame(std::shared_ptr // Get the frame's image std::shared_ptr frame_image = frame->GetImage(); - // Get original pixels for frame image, and also make a copy for editing - const unsigned char *original_pixels = (unsigned char *) frame_image->constBits(); - unsigned char *pixels = (unsigned char *) frame_image->bits(); + // Copy original pixels for reference, and get a writable pointer for editing + QImage original = frame_image->copy(); + const unsigned char *original_pixels = original.constBits(); + unsigned char *pixels = frame_image->bits(); int pixel_count = frame_image->width() * frame_image->height(); // Get current keyframe values @@ -77,7 +78,7 @@ std::shared_ptr Wave::GetFrame(std::shared_ptr float waveformVal = sin((Y * wavelength_value) + (time * speed_y_value)); // Waveform algorithm on y-axis float waveVal = (waveformVal + shift_x_value) * noiseAmp; // Shifts pixels on the x-axis - long unsigned int source_px = round(pixel + waveVal); + int source_px = lround(pixel + waveVal); if (source_px < 0) source_px = 0; if (source_px >= pixel_count) diff --git a/src/sort_filter/KalmanTracker.cpp b/src/sort_filter/KalmanTracker.cpp index 083f3b1ed..1a50e4c3b 100644 --- a/src/sort_filter/KalmanTracker.cpp +++ b/src/sort_filter/KalmanTracker.cpp @@ -15,23 +15,26 @@ using namespace cv; void KalmanTracker::init_kf( StateType stateMat) { - int stateNum = 7; + int stateNum = 8; int measureNum = 4; kf = KalmanFilter(stateNum, measureNum, 0); measurement = Mat::zeros(measureNum, 1, CV_32F); - kf.transitionMatrix = (Mat_(7, 7) << 1, 0, 0, 0, 1, 0, 0, + kf.transitionMatrix = (Mat_(8, 8) << 1, 0, 0, 0, 1, 0, 0, 0, - 0, 1, 0, 0, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 1, - 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 1); + 0, 1, 0, 0, 0, 1, 0, 0, + 0, 0, 1, 0, 0, 0, 1, 0, + 0, 0, 0, 1, 0, 0, 0, 1, + 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 0, 1); setIdentity(kf.measurementMatrix); setIdentity(kf.processNoiseCov, Scalar::all(1e-1)); + kf.processNoiseCov.at(2, 2) = 1e0; // higher noise for area (s) to adapt to size changes + kf.processNoiseCov.at(3, 3) = 1e0; // higher noise for aspect ratio (r) setIdentity(kf.measurementNoiseCov, Scalar::all(1e-4)); setIdentity(kf.errorCovPost, Scalar::all(1e-2)); @@ -40,6 +43,10 @@ void KalmanTracker::init_kf( kf.statePost.at(1, 0) = stateMat.y + stateMat.height / 2; kf.statePost.at(2, 0) = stateMat.area(); kf.statePost.at(3, 0) = stateMat.width / stateMat.height; + kf.statePost.at(4, 0) = 0.0f; + kf.statePost.at(5, 0) = 0.0f; + kf.statePost.at(6, 0) = 0.0f; + kf.statePost.at(7, 0) = 0.0f; } // Predict the estimated bounding box. diff --git a/src/sort_filter/sort.cpp b/src/sort_filter/sort.cpp index 611eeea42..78ae24320 100644 --- a/src/sort_filter/sort.cpp +++ b/src/sort_filter/sort.cpp @@ -7,10 +7,15 @@ using namespace std; // Constructor -SortTracker::SortTracker(int max_age, int min_hits) +SortTracker::SortTracker(int max_age, int min_hits, int max_missed, double min_iou, double nms_iou_thresh, double min_conf) { _min_hits = min_hits; _max_age = max_age; + _max_missed = max_missed; + _min_iou = min_iou; + _nms_iou_thresh = nms_iou_thresh; + _min_conf = min_conf; + _next_id = 0; alive_tracker = true; } @@ -42,6 +47,40 @@ double SortTracker::GetCentroidsDistance( return distance; } +// Function to apply NMS on detections +void apply_nms(vector& detections, double nms_iou_thresh) { + if (detections.empty()) return; + + // Sort detections by confidence descending + std::sort(detections.begin(), detections.end(), [](const TrackingBox& a, const TrackingBox& b) { + return a.confidence > b.confidence; + }); + + vector suppressed(detections.size(), false); + + for (size_t i = 0; i < detections.size(); ++i) { + if (suppressed[i]) continue; + + for (size_t j = i + 1; j < detections.size(); ++j) { + if (suppressed[j]) continue; + + if (detections[i].classId == detections[j].classId && + SortTracker::GetIOU(detections[i].box, detections[j].box) > nms_iou_thresh) { + suppressed[j] = true; + } + } + } + + // Remove suppressed detections + vector filtered; + for (size_t i = 0; i < detections.size(); ++i) { + if (!suppressed[i]) { + filtered.push_back(detections[i]); + } + } + detections = filtered; +} + void SortTracker::update(vector detections_cv, int frame_count, double image_diagonal, std::vector confidences, std::vector classIds) { vector detections; @@ -51,6 +90,8 @@ void SortTracker::update(vector detections_cv, int frame_count, double // initialize kalman trackers using first detections. for (unsigned int i = 0; i < detections_cv.size(); i++) { + if (confidences[i] < _min_conf) continue; // filter low conf + TrackingBox tb; tb.box = cv::Rect_(detections_cv[i]); @@ -58,7 +99,7 @@ void SortTracker::update(vector detections_cv, int frame_count, double tb.confidence = confidences[i]; detections.push_back(tb); - KalmanTracker trk = KalmanTracker(detections[i].box, detections[i].confidence, detections[i].classId, i); + KalmanTracker trk = KalmanTracker(detections.back().box, detections.back().confidence, detections.back().classId, _next_id++); trackers.push_back(trk); } return; @@ -67,12 +108,18 @@ void SortTracker::update(vector detections_cv, int frame_count, double { for (unsigned int i = 0; i < detections_cv.size(); i++) { + if (confidences[i] < _min_conf) continue; // filter low conf + TrackingBox tb; tb.box = cv::Rect_(detections_cv[i]); tb.classId = classIds[i]; tb.confidence = confidences[i]; detections.push_back(tb); } + + // Apply NMS to remove duplicates + apply_nms(detections, _nms_iou_thresh); + for (auto it = frameTrackingResult.begin(); it != frameTrackingResult.end(); it++) { int frame_age = frame_count - it->frame; @@ -101,22 +148,29 @@ void SortTracker::update(vector detections_cv, int frame_count, double trkNum = predictedBoxes.size(); detNum = detections.size(); - centroid_dist_matrix.clear(); - centroid_dist_matrix.resize(trkNum, vector(detNum, 0)); + cost_matrix.clear(); + cost_matrix.resize(trkNum, vector(detNum, 0)); - for (unsigned int i = 0; i < trkNum; i++) // compute iou matrix as a distance matrix + for (unsigned int i = 0; i < trkNum; i++) // compute cost matrix using 1 - IOU with gating { for (unsigned int j = 0; j < detNum; j++) { - // use 1-iou because the hungarian algorithm computes a minimum-cost assignment. - double distance = SortTracker::GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal; - centroid_dist_matrix[i][j] = distance; + double iou = GetIOU(predictedBoxes[i], detections[j].box); + double dist = GetCentroidsDistance(predictedBoxes[i], detections[j].box) / image_diagonal; + if (trackers[i].classId != detections[j].classId || dist > max_centroid_dist_norm) + { + cost_matrix[i][j] = 1e9; // large cost for gating + } + else + { + cost_matrix[i][j] = 1 - iou + (1 - detections[j].confidence) * 0.1; // slight penalty for low conf + } } } HungarianAlgorithm HungAlgo; assignment.clear(); - HungAlgo.Solve(centroid_dist_matrix, assignment); + HungAlgo.Solve(cost_matrix, assignment); // find matches, unmatched_detections and unmatched_predictions unmatchedTrajectories.clear(); unmatchedDetections.clear(); @@ -150,7 +204,7 @@ void SortTracker::update(vector detections_cv, int frame_count, double { if (assignment[i] == -1) // pass over invalid values continue; - if (centroid_dist_matrix[i][assignment[i]] > max_centroid_dist_norm) + if (cost_matrix[i][assignment[i]] > 1 - _min_iou) { unmatchedTrajectories.insert(i); unmatchedDetections.insert(assignment[i]); @@ -171,7 +225,7 @@ void SortTracker::update(vector detections_cv, int frame_count, double // create and initialise new trackers for unmatched detections for (auto umd : unmatchedDetections) { - KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId, umd); + KalmanTracker tracker = KalmanTracker(detections[umd].box, detections[umd].confidence, detections[umd].classId, _next_id++); trackers.push_back(tracker); } @@ -192,7 +246,8 @@ void SortTracker::update(vector detections_cv, int frame_count, double frameTrackingResult.clear(); for (unsigned int i = 0; i < trackers.size();) { - if ((trackers[i].m_time_since_update < 1 && trackers[i].m_hit_streak >= _min_hits) || frame_count <= _min_hits) + if ((trackers[i].m_hits >= _min_hits && trackers[i].m_time_since_update <= _max_missed) || + frame_count <= _min_hits) { alive_tracker = true; TrackingBox res; diff --git a/src/sort_filter/sort.hpp b/src/sort_filter/sort.hpp index 6d7f22e23..74e905adc 100644 --- a/src/sort_filter/sort.hpp +++ b/src/sort_filter/sort.hpp @@ -9,6 +9,7 @@ #include #include // to format image names using setw() and setfill() #include +#include // for std::sort #include "opencv2/video/tracking.hpp" #include "opencv2/highgui/highgui.hpp" @@ -25,7 +26,7 @@ typedef struct TrackingBox int classId = 0; int id = 0; cv::Rect_ box = cv::Rect_(0.0, 0.0, 0.0, 0.0); - TrackingBox() {} + TrackingBox() {} TrackingBox(int _frame, float _confidence, int _classId, int _id) : frame(_frame), confidence(_confidence), classId(_classId), id(_id) {} } TrackingBox; @@ -33,19 +34,19 @@ class SortTracker { public: // Constructor - SortTracker(int max_age = 7, int min_hits = 2); + SortTracker(int max_age = 50, int min_hits = 5, int max_missed = 7, double min_iou = 0.1, double nms_iou_thresh = 0.5, double min_conf = 0.3); // Initialize tracker // Update position based on the new frame void update(std::vector detection, int frame_count, double image_diagonal, std::vector confidences, std::vector classIds); - double GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt); + static double GetIOU(cv::Rect_ bb_test, cv::Rect_ bb_gt); double GetCentroidsDistance(cv::Rect_ bb_test, cv::Rect_ bb_gt); std::vector trackers; - double max_centroid_dist_norm = 0.05; + double max_centroid_dist_norm = 0.3; std::vector> predictedBoxes; - std::vector> centroid_dist_matrix; + std::vector> cost_matrix; std::vector assignment; std::set unmatchedDetections; std::set unmatchedTrajectories; @@ -60,5 +61,10 @@ class SortTracker unsigned int detNum = 0; int _min_hits; int _max_age; + int _max_missed; + double _min_iou; + double _nms_iou_thresh; + double _min_conf; + unsigned int _next_id; bool alive_tracker; }; diff --git a/tests/AnalogTape.cpp b/tests/AnalogTape.cpp new file mode 100644 index 000000000..1e36f0e73 --- /dev/null +++ b/tests/AnalogTape.cpp @@ -0,0 +1,90 @@ +/** + * @file + * @brief Unit tests for AnalogTape effect + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include + +#include "Frame.h" +#include "effects/AnalogTape.h" +#include "openshot_catch.h" + +using namespace openshot; + +// Fixed helper ensures Frame invariants are respected (size/format/flags) +static std::shared_ptr makeGrayFrame(int w = 64, int h = 64) { + auto f = std::make_shared(1, w, h, "#000000", 0, 2); + + // Use premultiplied format to match Frame::AddImage expectations + auto img = std::make_shared(w, h, QImage::Format_RGBA8888_Premultiplied); + img->fill(QColor(100, 100, 100, 255)); + + // Route through AddImage so width/height/has_image_data are set correctly + f->AddImage(img); + return f; +} + +TEST_CASE("AnalogTape modifies frame", "[effect][analogtape]") { + AnalogTape eff; + eff.Id("analogtape-test-seed"); + eff.seed_offset = 1234; + auto frame = makeGrayFrame(); + QColor before = frame->GetImage()->pixelColor(2, 2); + auto out = eff.GetFrame(frame, 1); + QColor after = out->GetImage()->pixelColor(2, 2); + CHECK(after != before); +} + +TEST_CASE("AnalogTape deterministic per id", "[effect][analogtape]") { + AnalogTape e1; + e1.Id("same"); + AnalogTape e2; + e2.Id("same"); + auto f1 = makeGrayFrame(); + auto f2 = makeGrayFrame(); + auto o1 = e1.GetFrame(f1, 1); + auto o2 = e2.GetFrame(f2, 1); + QColor c1 = o1->GetImage()->pixelColor(1, 1); + QColor c2 = o2->GetImage()->pixelColor(1, 1); + CHECK(c1 == c2); +} + +TEST_CASE("AnalogTape seed offset alters output", "[effect][analogtape]") { + AnalogTape e1; + e1.Id("seed"); + e1.seed_offset = 0; + AnalogTape e2; + e2.Id("seed"); + e2.seed_offset = 5; + auto f1 = makeGrayFrame(); + auto f2 = makeGrayFrame(); + auto o1 = e1.GetFrame(f1, 1); + auto o2 = e2.GetFrame(f2, 1); + QColor c1 = o1->GetImage()->pixelColor(1, 1); + QColor c2 = o2->GetImage()->pixelColor(1, 1); + CHECK(c1 != c2); +} + +TEST_CASE("AnalogTape stripe lifts bottom", "[effect][analogtape]") { + AnalogTape e; + e.tracking = Keyframe(0.0); + e.bleed = Keyframe(0.0); + e.softness = Keyframe(0.0); + e.noise = Keyframe(0.0); + e.stripe = Keyframe(1.0); + e.staticBands = Keyframe(0.0); + auto frame = makeGrayFrame(20, 20); + auto out = e.GetFrame(frame, 1); + QColor top = out->GetImage()->pixelColor(10, 0); + QColor bottom = out->GetImage()->pixelColor(10, 19); + CHECK(bottom.red() > top.red()); +} diff --git a/tests/AudioWaveformer.cpp b/tests/AudioWaveformer.cpp index c8d856836..7327c3835 100644 --- a/tests/AudioWaveformer.cpp +++ b/tests/AudioWaveformer.cpp @@ -12,193 +12,447 @@ #include "openshot_catch.h" #include "AudioWaveformer.h" +#include "Clip.h" +#include "Exceptions.h" #include "FFmpegReader.h" +#include "Timeline.h" + +#include +#include +#include +#include +#include using namespace openshot; TEST_CASE( "Extract waveform data piano.wav", "[libopenshot][audiowaveformer]" ) { - // Create a reader - std::stringstream path; - path << TEST_MEDIA_PATH << "piano.wav"; - FFmpegReader r(path.str()); - r.Open(); - - // Create AudioWaveformer and extract a smaller "average" sample set of audio data - AudioWaveformer waveformer(&r); - for (auto channel = 0; channel < r.info.channels; channel++) { - AudioWaveformData waveform = waveformer.ExtractSamples(channel, 20, false); - - if (channel == 0) { - CHECK(waveform.rms_samples.size() == 107); - CHECK(waveform.rms_samples[0] == Approx(0.04879f).margin(0.00001)); - CHECK(waveform.rms_samples[86] == Approx(0.13578f).margin(0.00001)); - CHECK(waveform.rms_samples[87] == Approx(0.0f).margin(0.00001)); - } else if (channel == 1) { - CHECK(waveform.rms_samples.size() == 107); - CHECK(waveform.rms_samples[0] == Approx(0.04879f).margin(0.00001)); - CHECK(waveform.rms_samples[86] == Approx(0.13578f).margin(0.00001)); - CHECK(waveform.rms_samples[87] == Approx(0.0f).margin(0.00001)); - } - - waveform.clear(); - } - - // Clean up - r.Close(); + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "piano.wav"; + FFmpegReader r(path.str()); + r.Open(); + + // Create AudioWaveformer and extract a smaller "average" sample set of audio data + const int samples_per_second = 20; + const int expected_total = static_cast(std::ceil(r.info.duration * samples_per_second)); + REQUIRE(expected_total > 1); + + AudioWaveformer waveformer(&r); + for (auto channel = 0; channel < r.info.channels; channel++) { + AudioWaveformData waveform = waveformer.ExtractSamples(channel, samples_per_second, false); + + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(waveform.rms_samples[0] >= 0.0f); + CHECK(waveform.rms_samples.back() >= 0.0f); + CHECK(*std::max_element(waveform.rms_samples.begin(), waveform.rms_samples.end()) > 0.0f); + + waveform.clear(); + } + + // Clean up + r.Close(); } TEST_CASE( "Extract waveform data sintel", "[libopenshot][audiowaveformer]" ) { - // Create a reader - std::stringstream path; - path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; - FFmpegReader r(path.str()); - - // Create AudioWaveformer and extract a smaller "average" sample set of audio data - AudioWaveformer waveformer(&r); - for (auto channel = 0; channel < r.info.channels; channel++) { - AudioWaveformData waveform = waveformer.ExtractSamples(channel, 20, false); - - if (channel == 0) { - CHECK(waveform.rms_samples.size() == 1058); - CHECK(waveform.rms_samples[0] == Approx(0.00001f).margin(0.00001)); - CHECK(waveform.rms_samples[1037] == Approx(0.00003f).margin(0.00001)); - CHECK(waveform.rms_samples[1038] == Approx(0.0f).margin(0.00001)); - } else if (channel == 1) { - CHECK(waveform.rms_samples.size() == 1058); - CHECK(waveform.rms_samples[0] == Approx(0.00001f ).margin(0.00001)); - CHECK(waveform.rms_samples[1037] == Approx(0.00003f).margin(0.00001)); - CHECK(waveform.rms_samples[1038] == Approx(0.0f).margin(0.00001)); - } - - waveform.clear(); - } - - // Clean up - r.Close(); + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str()); + + // Create AudioWaveformer and extract a smaller "average" sample set of audio data + const int samples_per_second = 20; + const int expected_total = static_cast(std::ceil(r.info.duration * samples_per_second)); + REQUIRE(expected_total > 1); + + AudioWaveformer waveformer(&r); + for (auto channel = 0; channel < r.info.channels; channel++) { + AudioWaveformData waveform = waveformer.ExtractSamples(channel, samples_per_second, false); + + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(waveform.rms_samples[0] >= 0.0f); + CHECK(waveform.rms_samples.back() >= 0.0f); + + waveform.clear(); + } + + // Clean up + r.Close(); } TEST_CASE( "Extract waveform data sintel (all channels)", "[libopenshot][audiowaveformer]" ) { - // Create a reader - std::stringstream path; - path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; - FFmpegReader r(path.str()); + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str()); + + // Create AudioWaveformer and extract a smaller "average" sample set of audio data + const int samples_per_second = 20; + const int expected_total = static_cast(std::ceil(r.info.duration * samples_per_second)); + REQUIRE(expected_total > 1); + + AudioWaveformer waveformer(&r); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(waveform.rms_samples[0] >= 0.0f); + CHECK(waveform.rms_samples.back() >= 0.0f); + + waveform.clear(); + + // Clean up + r.Close(); +} + +TEST_CASE( "Channel selection returns data and rejects invalid channel", "[libopenshot][audiowaveformer][channels]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str()); + r.Open(); + + AudioWaveformer waveformer(&r); + const int samples_per_second = 20; + const int expected_total = static_cast(std::ceil(r.info.duration * samples_per_second)); + + AudioWaveformData ch0 = waveformer.ExtractSamples(0, samples_per_second, false); + AudioWaveformData ch1 = waveformer.ExtractSamples(1, samples_per_second, false); + AudioWaveformData all = waveformer.ExtractSamples(-1, samples_per_second, false); + + CHECK(ch0.rms_samples.size() == expected_total); + CHECK(ch1.rms_samples.size() == expected_total); + CHECK(all.rms_samples.size() == expected_total); + + // All-channels max should be at least as large as per-channel max + CHECK(*std::max_element(all.max_samples.begin(), all.max_samples.end()) >= *std::max_element(ch0.max_samples.begin(), ch0.max_samples.end())); + CHECK(*std::max_element(all.max_samples.begin(), all.max_samples.end()) >= *std::max_element(ch1.max_samples.begin(), ch1.max_samples.end())); + + // Out of range channel returns empty data + AudioWaveformData invalid = waveformer.ExtractSamples(10, samples_per_second, false); + CHECK(invalid.rms_samples.empty()); + + r.Close(); +} + +TEST_CASE( "Waveform extraction does not mutate source reader video flag", "[libopenshot][audiowaveformer][mutation]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + Clip clip(&reader); + clip.Open(); + + const bool original_has_video_clip = clip.Reader()->info.has_video; + const bool original_has_video_reader = reader.info.has_video; + REQUIRE(original_has_video_clip == original_has_video_reader); + REQUIRE(original_has_video_reader); + + AudioWaveformer waveformer(&clip); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, 5, false); + + // Extraction should not flip has_video on the live reader/clip + CHECK_FALSE(waveform.rms_samples.empty()); + CHECK(clip.Reader()->info.has_video == original_has_video_clip); + CHECK(reader.info.has_video == original_has_video_reader); + + clip.Close(); + reader.Close(); +} + + +TEST_CASE( "Extract waveform waits for reader reopen", "[libopenshot][audiowaveformer][stability]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + + AudioWaveformer waveformer(&reader); + const int samples_per_second = 20; + + auto future_waveform = std::async(std::launch::async, [&]() { + return waveformer.ExtractSamples(-1, samples_per_second, false); + }); + + reader.Close(); + reader.Open(); + reader.Close(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + reader.Open(); + + AudioWaveformData waveform; + REQUIRE_NOTHROW(waveform = future_waveform.get()); + CHECK_FALSE(waveform.rms_samples.empty()); + + reader.Close(); +} + +TEST_CASE( "Extract waveform continues if caller closes original reader", "[libopenshot][audiowaveformer][stability]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); - // Create AudioWaveformer and extract a smaller "average" sample set of audio data - AudioWaveformer waveformer(&r); - AudioWaveformData waveform = waveformer.ExtractSamples(-1, 20, false); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 20; - CHECK(waveform.rms_samples.size() == 1058); - CHECK(waveform.rms_samples[0] == Approx(0.00001f).margin(0.00001)); - CHECK(waveform.rms_samples[1037] == Approx(0.00003f).margin(0.00001)); - CHECK(waveform.rms_samples[1038] == Approx(0.0f).margin(0.00001)); + auto future_waveform = std::async(std::launch::async, [&]() { + return waveformer.ExtractSamples(-1, samples_per_second, false); + }); - waveform.clear(); + // Closing the caller's reader should not affect a detached clone used for waveform extraction. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + reader.Close(); - // Clean up - r.Close(); + AudioWaveformData waveform; + REQUIRE_NOTHROW(waveform = future_waveform.get()); + CHECK_FALSE(waveform.rms_samples.empty()); + + reader.Close(); } TEST_CASE( "Normalize & scale waveform data piano.wav", "[libopenshot][audiowaveformer]" ) { - // Create a reader - std::stringstream path; - path << TEST_MEDIA_PATH << "piano.wav"; - FFmpegReader r(path.str()); - - // Create AudioWaveformer and extract a smaller "average" sample set of audio data - AudioWaveformer waveformer(&r); - for (auto channel = 0; channel < r.info.channels; channel++) { - // Normalize values and scale them between -1 and +1 - AudioWaveformData waveform = waveformer.ExtractSamples(channel, 20, true); - - if (channel == 0) { - CHECK(waveform.rms_samples.size() == 107); - CHECK(waveform.rms_samples[0] == Approx(0.07524f).margin(0.00001)); - CHECK(waveform.rms_samples[35] == Approx(0.20063f).margin(0.00001)); - CHECK(waveform.rms_samples[86] == Approx(0.2094f).margin(0.00001)); - CHECK(waveform.rms_samples[87] == Approx(0.0f).margin(0.00001)); - } - - waveform.clear(); - } - - // Clean up - r.Close(); + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "piano.wav"; + FFmpegReader r(path.str()); + + // Create AudioWaveformer and extract a smaller "average" sample set of audio data + const int samples_per_second = 20; + const int expected_total = static_cast(std::ceil(r.info.duration * samples_per_second)); + REQUIRE(expected_total > 1); + + AudioWaveformer waveformer(&r); + for (auto channel = 0; channel < r.info.channels; channel++) { + // Normalize values and scale them between -1 and +1 + AudioWaveformData waveform = waveformer.ExtractSamples(channel, samples_per_second, true); + + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(waveform.rms_samples[0] >= 0.0f); + CHECK(waveform.rms_samples.back() >= 0.0f); + CHECK(*std::max_element(waveform.max_samples.begin(), waveform.max_samples.end()) <= Approx(1.0f).margin(0.0001f)); + + waveform.clear(); + } + + // Clean up + r.Close(); +} + +TEST_CASE( "Extract waveform data clip slowed by time curve", "[libopenshot][audiowaveformer][clip][time]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + + FFmpegReader reader(path.str()); + Clip clip(&reader); + clip.Open(); + + const int64_t original_video_length = clip.Reader()->info.video_length; + const double fps_value = clip.Reader()->info.fps.ToDouble(); + REQUIRE(original_video_length > 0); + REQUIRE(fps_value > 0.0); + + clip.time = Keyframe(); + clip.time.AddPoint(1.0, 1.0, LINEAR); + clip.time.AddPoint(static_cast(original_video_length) * 2.0, + static_cast(original_video_length), LINEAR); + + AudioWaveformer waveformer(&clip); + const int samples_per_second = 20; + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + const double expected_duration = (static_cast(original_video_length) * 2.0) / fps_value; + const int expected_total = static_cast(std::ceil(expected_duration * samples_per_second)); + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(clip.time.GetLength() == original_video_length * 2); + CHECK(clip.time.GetLength() == static_cast(std::llround(expected_duration * fps_value))); + + clip.Close(); + reader.Close(); +} + +TEST_CASE( "Extract waveform data clip reversed by time curve", "[libopenshot][audiowaveformer][clip][time]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "piano.wav"; + + FFmpegReader reader(path.str()); + Clip clip(&reader); + clip.Open(); + + const int samples_per_second = 20; + const int base_total = static_cast(std::ceil(clip.Reader()->info.duration * samples_per_second)); + const int64_t original_video_length = clip.Reader()->info.video_length; + const double fps_value = clip.Reader()->info.fps.ToDouble(); + REQUIRE(original_video_length > 0); + REQUIRE(fps_value > 0.0); + + clip.time = Keyframe(); + clip.time.AddPoint(1.0, static_cast(original_video_length), LINEAR); + clip.time.AddPoint(static_cast(original_video_length), 1.0, LINEAR); + + AudioWaveformer waveformer(&clip); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + const double expected_duration = static_cast(original_video_length) / fps_value; + const int expected_total = static_cast(std::ceil(expected_duration * samples_per_second)); + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(expected_total == base_total); + CHECK(clip.time.GetLength() == original_video_length); + CHECK(clip.time.GetLength() == static_cast(std::llround(expected_duration * fps_value))); + + clip.Close(); + reader.Close(); +} + +TEST_CASE( "Extract waveform data clip reversed and slowed", "[libopenshot][audiowaveformer][clip][time]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "piano.wav"; + + FFmpegReader reader(path.str()); + Clip clip(&reader); + clip.Open(); + + const int samples_per_second = 20; + const int base_total = static_cast(std::ceil(clip.Reader()->info.duration * samples_per_second)); + const int64_t original_video_length = clip.Reader()->info.video_length; + const double fps_value = clip.Reader()->info.fps.ToDouble(); + REQUIRE(original_video_length > 0); + REQUIRE(fps_value > 0.0); + + clip.time = Keyframe(); + clip.time.AddPoint(1.0, static_cast(original_video_length), LINEAR); + clip.time.AddPoint(static_cast(original_video_length) * 2.0, 1.0, LINEAR); + + AudioWaveformer waveformer(&clip); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + const double expected_duration = (static_cast(original_video_length) * 2.0) / fps_value; + const int expected_total = static_cast(std::ceil(expected_duration * samples_per_second)); + CHECK(waveform.rms_samples.size() == expected_total); + CHECK(expected_total > base_total); + CHECK(clip.time.GetLength() == original_video_length * 2); + CHECK(clip.time.GetLength() == static_cast(std::llround(expected_duration * fps_value))); + + clip.Close(); + reader.Close(); +} + +TEST_CASE( "Clip duration uses parent timeline FPS when time-mapped", "[libopenshot][audiowaveformer][clip][time][timeline]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "piano.wav"; + + FFmpegReader reader(path.str()); + Clip clip(&reader); + clip.Open(); + + const int64_t original_video_length = clip.Reader()->info.video_length; + const double reader_fps = clip.Reader()->info.fps.ToDouble(); + REQUIRE(original_video_length > 0); + REQUIRE(reader_fps > 0.0); + + Timeline timeline( + 640, + 480, + Fraction(60, 1), + clip.Reader()->info.sample_rate, + clip.Reader()->info.channels, + clip.Reader()->info.channel_layout); + + clip.ParentTimeline(&timeline); + + clip.time = Keyframe(); + clip.time.AddPoint(1.0, 1.0, LINEAR); + clip.time.AddPoint(static_cast(original_video_length) * 2.0, + static_cast(original_video_length), LINEAR); + + const double timeline_fps = timeline.info.fps.ToDouble(); + REQUIRE(timeline_fps > 0.0); + + const double expected_duration = (static_cast(original_video_length) * 2.0) / timeline_fps; + CHECK(clip.time.GetLength() == static_cast(std::llround(expected_duration * timeline_fps))); + + clip.Close(); + reader.Close(); } TEST_CASE( "Extract waveform from image (no audio)", "[libopenshot][audiowaveformer]" ) { - // Create a reader - std::stringstream path; - path << TEST_MEDIA_PATH << "front.png"; - FFmpegReader r(path.str()); + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader r(path.str()); - // Create AudioWaveformer and extract a smaller "average" sample set of audio data - AudioWaveformer waveformer(&r); - AudioWaveformData waveform = waveformer.ExtractSamples(-1, 20, false); + // Create AudioWaveformer and extract a smaller "average" sample set of audio data + AudioWaveformer waveformer(&r); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, 20, false); - CHECK(waveform.rms_samples.size() == 0); - CHECK(waveform.max_samples.size() == 0); + CHECK(waveform.rms_samples.size() == 0); + CHECK(waveform.max_samples.size() == 0); - // Clean up - r.Close(); + // Clean up + r.Close(); } TEST_CASE( "AudioWaveformData struct methods", "[libopenshot][audiowaveformer]" ) { - // Create a reader - AudioWaveformData waveform; - - // Resize data to 10 elements - waveform.resize(10); - CHECK(waveform.rms_samples.size() == 10); - CHECK(waveform.max_samples.size() == 10); - - // Set all values = 1.0 - for (auto s = 0; s < waveform.rms_samples.size(); s++) { - waveform.rms_samples[s] = 1.0; - waveform.max_samples[s] = 1.0; - } - CHECK(waveform.rms_samples[0] == Approx(1.0f).margin(0.00001)); - CHECK(waveform.rms_samples[9] == Approx(1.0f).margin(0.00001)); - CHECK(waveform.max_samples[0] == Approx(1.0f).margin(0.00001)); - CHECK(waveform.max_samples[9] == Approx(1.0f).margin(0.00001)); - - // Scale all values by 2 - waveform.scale(10, 2.0); - CHECK(waveform.rms_samples.size() == 10); - CHECK(waveform.max_samples.size() == 10); - CHECK(waveform.rms_samples[0] == Approx(2.0f).margin(0.00001)); - CHECK(waveform.rms_samples[9] == Approx(2.0f).margin(0.00001)); - CHECK(waveform.max_samples[0] == Approx(2.0f).margin(0.00001)); - CHECK(waveform.max_samples[9] == Approx(2.0f).margin(0.00001)); - - // Zero out all values - waveform.zero(10); - CHECK(waveform.rms_samples.size() == 10); - CHECK(waveform.max_samples.size() == 10); - CHECK(waveform.rms_samples[0] == Approx(0.0f).margin(0.00001)); - CHECK(waveform.rms_samples[9] == Approx(0.0f).margin(0.00001)); - CHECK(waveform.max_samples[0] == Approx(0.0f).margin(0.00001)); - CHECK(waveform.max_samples[9] == Approx(0.0f).margin(0.00001)); - - // Access vectors and verify size - std::vector> vectors = waveform.vectors(); - CHECK(vectors.size() == 2); - CHECK(vectors[0].size() == 10); - CHECK(vectors[0].size() == 10); - - // Clear and verify internal data is empty - waveform.clear(); - CHECK(waveform.rms_samples.size() == 0); - CHECK(waveform.max_samples.size() == 0); - vectors = waveform.vectors(); - CHECK(vectors.size() == 2); - CHECK(vectors[0].size() == 0); - CHECK(vectors[0].size() == 0); + // Create a reader + AudioWaveformData waveform; + + // Resize data to 10 elements + waveform.resize(10); + CHECK(waveform.rms_samples.size() == 10); + CHECK(waveform.max_samples.size() == 10); + + // Set all values = 1.0 + for (auto s = 0; s < waveform.rms_samples.size(); s++) { + waveform.rms_samples[s] = 1.0; + waveform.max_samples[s] = 1.0; + } + CHECK(waveform.rms_samples[0] == Approx(1.0f).margin(0.00001)); + CHECK(waveform.rms_samples[9] == Approx(1.0f).margin(0.00001)); + CHECK(waveform.max_samples[0] == Approx(1.0f).margin(0.00001)); + CHECK(waveform.max_samples[9] == Approx(1.0f).margin(0.00001)); + + // Scale all values by 2 + waveform.scale(10, 2.0); + CHECK(waveform.rms_samples.size() == 10); + CHECK(waveform.max_samples.size() == 10); + CHECK(waveform.rms_samples[0] == Approx(2.0f).margin(0.00001)); + CHECK(waveform.rms_samples[9] == Approx(2.0f).margin(0.00001)); + CHECK(waveform.max_samples[0] == Approx(2.0f).margin(0.00001)); + CHECK(waveform.max_samples[9] == Approx(2.0f).margin(0.00001)); + + // Zero out all values + waveform.zero(10); + CHECK(waveform.rms_samples.size() == 10); + CHECK(waveform.max_samples.size() == 10); + CHECK(waveform.rms_samples[0] == Approx(0.0f).margin(0.00001)); + CHECK(waveform.rms_samples[9] == Approx(0.0f).margin(0.00001)); + CHECK(waveform.max_samples[0] == Approx(0.0f).margin(0.00001)); + CHECK(waveform.max_samples[9] == Approx(0.0f).margin(0.00001)); + + // Access vectors and verify size + std::vector> vectors = waveform.vectors(); + CHECK(vectors.size() == 2); + CHECK(vectors[0].size() == 10); + CHECK(vectors[0].size() == 10); + + // Clear and verify internal data is empty + waveform.clear(); + CHECK(waveform.rms_samples.size() == 0); + CHECK(waveform.max_samples.size() == 0); + vectors = waveform.vectors(); + CHECK(vectors.size() == 2); + CHECK(vectors[0].size() == 0); + CHECK(vectors[0].size() == 0); } diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp new file mode 100644 index 000000000..669542574 --- /dev/null +++ b/tests/Benchmark.cpp @@ -0,0 +1,226 @@ +/** + * @file + * @brief Benchmark executable for core libopenshot operations + * @author Jonathan Thomas + * @ref License + */ +// Copyright (c) 2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include + +#include "Clip.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "Fraction.h" +#include "FrameMapper.h" +#ifdef USE_IMAGEMAGICK +#include "ImageReader.h" +#else +#include "QtImageReader.h" +#endif +#include "ReaderBase.h" +#include "Timeline.h" +#include "effects/Brightness.h" +#include "effects/Crop.h" +#include "effects/Mask.h" +#include "effects/Saturation.h" + +using namespace openshot; +using namespace std; + +using Clock = chrono::steady_clock; + +template double time_trial(const string &name, Func func) { + auto start = Clock::now(); + func(); + auto elapsed = + chrono::duration_cast(Clock::now() - start).count(); + cout << name << "," << elapsed << "\n"; + return static_cast(elapsed); +} + +void read_forward_backward(ReaderBase &reader) { + int64_t len = reader.info.video_length; + for (int64_t i = 1; i <= len; ++i) + reader.GetFrame(i); + for (int64_t i = len; i >= 1; --i) + reader.GetFrame(i); +} + +int main() { + cout << "Trial,Milliseconds\n"; + double total = 0.0; + const string base = TEST_MEDIA_PATH; + const string video = base + "sintel_trailer-720p.mp4"; + const string mask_img = base + "mask.png"; + const string overlay = base + "front3.png"; + + total += time_trial("FFmpegReader", [&]() { + FFmpegReader r(video); + r.Open(); + read_forward_backward(r); + r.Close(); + }); + + total += time_trial("FFmpegWriter", [&]() { + FFmpegReader r(video); + r.Open(); + FFmpegWriter w("benchmark_output.mp4"); + w.SetAudioOptions("aac", r.info.sample_rate, 192000); + w.SetVideoOptions("libx264", r.info.width, r.info.height, r.info.fps, + 5000000); + w.Open(); + for (int64_t i = 1; i <= r.info.video_length; ++i) + w.WriteFrame(r.GetFrame(i)); + w.Close(); + r.Close(); + }); + + total += time_trial("FrameMapper", [&]() { + vector rates = {Fraction(24, 1), Fraction(30, 1), Fraction(60, 1), + Fraction(30000, 1001), Fraction(60000, 1001)}; + for (auto &fps : rates) { + FFmpegReader r(video); + r.Open(); + FrameMapper map(&r, fps, PULLDOWN_NONE, r.info.sample_rate, + r.info.channels, r.info.channel_layout); + map.Open(); + for (int64_t i = 1; i <= map.info.video_length; ++i) + map.GetFrame(i); + map.Close(); + r.Close(); + } + }); + + total += time_trial("Clip", [&]() { + Clip c(video); + c.Open(); + read_forward_backward(c); + c.Close(); + }); + + total += time_trial("Timeline", [&]() { + Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); + Clip video_clip(video); + video_clip.Layer(0); + video_clip.Start(0.0); + video_clip.End(video_clip.Reader()->info.duration); + video_clip.Open(); + Clip overlay1(overlay); + overlay1.Layer(1); + overlay1.Start(0.0); + overlay1.End(video_clip.Reader()->info.duration); + overlay1.Open(); + Clip overlay2(overlay); + overlay2.Layer(2); + overlay2.Start(0.0); + overlay2.End(video_clip.Reader()->info.duration); + overlay2.Open(); + t.AddClip(&video_clip); + t.AddClip(&overlay1); + t.AddClip(&overlay2); + t.Open(); + t.info.video_length = t.GetMaxFrame(); + read_forward_backward(t); + t.Close(); + }); + + total += time_trial("Timeline (with transforms)", [&]() { + Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); + Clip video_clip(video); + int64_t last = video_clip.Reader()->info.video_length; + video_clip.Layer(0); + video_clip.Start(0.0); + video_clip.End(video_clip.Reader()->info.duration); + video_clip.alpha.AddPoint(1, 1.0); + video_clip.alpha.AddPoint(last, 0.0); + video_clip.Open(); + Clip overlay1(overlay); + overlay1.Layer(1); + overlay1.Start(0.0); + overlay1.End(video_clip.Reader()->info.duration); + overlay1.Open(); + overlay1.scale_x.AddPoint(1, 1.0); + overlay1.scale_x.AddPoint(last, 0.25); + overlay1.scale_y.AddPoint(1, 1.0); + overlay1.scale_y.AddPoint(last, 0.25); + Clip overlay2(overlay); + overlay2.Layer(2); + overlay2.Start(0.0); + overlay2.End(video_clip.Reader()->info.duration); + overlay2.Open(); + overlay2.rotation.AddPoint(1, 90.0); + t.AddClip(&video_clip); + t.AddClip(&overlay1); + t.AddClip(&overlay2); + t.Open(); + t.info.video_length = t.GetMaxFrame(); + read_forward_backward(t); + t.Close(); + }); + + total += time_trial("Effect_Mask", [&]() { + FFmpegReader r(video); + r.Open(); +#ifdef USE_IMAGEMAGICK + ImageReader mask_reader(mask_img); +#else + QtImageReader mask_reader(mask_img); +#endif + mask_reader.Open(); + Clip clip(&r); + clip.Open(); + Mask m(&mask_reader, Keyframe(0.0), Keyframe(0.5)); + clip.AddEffect(&m); + read_forward_backward(clip); + mask_reader.Close(); + clip.Close(); + r.Close(); + }); + + total += time_trial("Effect_Brightness", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + Brightness b(Keyframe(0.5), Keyframe(1.0)); + clip.AddEffect(&b); + read_forward_backward(clip); + clip.Close(); + r.Close(); + }); + + total += time_trial("Effect_Crop", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + Crop c(Keyframe(0.25), Keyframe(0.25), Keyframe(0.25), Keyframe(0.25)); + clip.AddEffect(&c); + read_forward_backward(clip); + clip.Close(); + r.Close(); + }); + + total += time_trial("Effect_Saturation", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + Saturation s(Keyframe(0.25), Keyframe(0.25), Keyframe(0.25), + Keyframe(0.25)); + clip.AddEffect(&s); + read_forward_backward(clip); + clip.Close(); + r.Close(); + }); + + cout << "Overall," << total << "\n"; + return 0; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c24b1f617..50d71b3f4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,6 +16,11 @@ endif() # Test media path, used by unit tests for input data file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/examples/" TEST_MEDIA_PATH) +# Benchmark executable +add_executable(openshot-benchmark Benchmark.cpp) +target_compile_definitions(openshot-benchmark PRIVATE -DTEST_MEDIA_PATH="${TEST_MEDIA_PATH}") +target_link_libraries(openshot-benchmark openshot) + ### ### TEST SOURCE FILES ### @@ -48,8 +53,10 @@ set(OPENSHOT_TESTS ChromaKey Crop LensFlare + AnalogTape Sharpen SphericalEffect + WaveEffect ) # ImageMagick related test files diff --git a/tests/CVTracker.cpp b/tests/CVTracker.cpp index 95bcc6c8d..0ac1116c5 100644 --- a/tests/CVTracker.cpp +++ b/tests/CVTracker.cpp @@ -107,6 +107,43 @@ TEST_CASE( "Track_Video", "[libopenshot][opencv][tracker]" ) CHECK(height == Approx(166).margin(2)); } +TEST_CASE( "Track_BoundingBoxClipping", "[libopenshot][opencv][tracker]" ) +{ + // Create a video clip + std::stringstream path; + path << TEST_MEDIA_PATH << "test.avi"; + + // Open clip + openshot::Clip c1(path.str()); + c1.Open(); + + std::string json_data = R"proto( + { + "tracker-type": "KCF", + "region": { + "normalized_x": -0.2, + "normalized_y": -0.2, + "normalized_width": 1.5, + "normalized_height": 1.5, + "first-frame": 1 + } + } )proto"; + + ProcessingController tracker_pc; + CVTracker tracker(json_data, tracker_pc); + tracker_pc.SetError(false, ""); + + // Grab first frame and run tracker directly + std::shared_ptr f = c1.GetFrame(1); + cv::Mat image = f->GetImageCV(); + + tracker.initTracker(image, 1); + tracker.trackFrame(image, 2); + + INFO(tracker_pc.GetErrorMessage()); + CHECK(tracker_pc.GetError() == false); +} + TEST_CASE( "SaveLoad_Protobuf", "[libopenshot][opencv][tracker]" ) { diff --git a/tests/Clip.cpp b/tests/Clip.cpp index a6e3e6299..53500d623 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -12,14 +12,21 @@ #include #include +#include #include "openshot_catch.h" #include #include #include +#include +#include +#include #include "Clip.h" + +#include + #include "DummyReader.h" #include "Enums.h" #include "Exceptions.h" @@ -42,6 +49,7 @@ TEST_CASE( "default constructor", "[libopenshot][clip]" ) CHECK(c1.anchor == ANCHOR_CANVAS); CHECK(c1.gravity == GRAVITY_CENTER); CHECK(c1.scale == SCALE_FIT); + CHECK(c1.composite == COMPOSITE_SOURCE_OVER); CHECK(c1.Layer() == 0); CHECK(c1.Position() == Approx(0.0f).margin(0.00001)); CHECK(c1.Start() == Approx(0.0f).margin(0.00001)); @@ -60,10 +68,11 @@ TEST_CASE( "path string constructor", "[libopenshot][clip]" ) CHECK(c1.anchor == ANCHOR_CANVAS); CHECK(c1.gravity == GRAVITY_CENTER); CHECK(c1.scale == SCALE_FIT); + CHECK(c1.composite == COMPOSITE_SOURCE_OVER); CHECK(c1.Layer() == 0); CHECK(c1.Position() == Approx(0.0f).margin(0.00001)); CHECK(c1.Start() == Approx(0.0f).margin(0.00001)); - CHECK(c1.End() == Approx(4.39937f).margin(0.00001)); + CHECK(c1.End() == Approx(4.4).margin(0.00001)); } TEST_CASE( "basic getters and setters", "[libopenshot][clip]" ) @@ -76,6 +85,7 @@ TEST_CASE( "basic getters and setters", "[libopenshot][clip]" ) CHECK(c1.anchor == ANCHOR_CANVAS); CHECK(c1.gravity == GRAVITY_CENTER); CHECK(c1.scale == SCALE_FIT); + CHECK(c1.composite == COMPOSITE_SOURCE_OVER); CHECK(c1.Layer() == 0); CHECK(c1.Position() == Approx(0.0f).margin(0.00001)); CHECK(c1.Start() == Approx(0.0f).margin(0.00001)); @@ -172,6 +182,49 @@ TEST_CASE( "properties", "[libopenshot][clip]" ) delete reader; } +TEST_CASE( "Metadata rotation does not override manual scaling", "[libopenshot][clip]" ) +{ + DummyReader reader(Fraction(24, 1), 640, 480, 48000, 2, 5.0f); + Clip clip; + clip.scale_x = Keyframe(0.5); + clip.scale_y = Keyframe(0.5); + + clip.Reader(&reader); + + + REQUIRE(clip.rotation.GetCount() == 1); + CHECK(clip.rotation.GetPoint(0).co.Y == Approx(0.0).margin(0.00001)); + CHECK(clip.scale_x.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); + CHECK(clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); +} + +TEST_CASE( "Metadata rotation scales only default clips", "[libopenshot][clip]" ) +{ + DummyReader rotated(Fraction(24, 1), 640, 480, 48000, 2, 5.0f); + rotated.info.metadata["rotate"] = "90"; + + Clip auto_clip; + auto_clip.Reader(&rotated); + + REQUIRE(auto_clip.rotation.GetCount() == 1); + CHECK(auto_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001)); + CHECK(auto_clip.scale_x.GetPoint(0).co.Y == Approx(0.75).margin(0.00001)); + CHECK(auto_clip.scale_y.GetPoint(0).co.Y == Approx(0.75).margin(0.00001)); + + DummyReader rotated_custom(Fraction(24, 1), 640, 480, 48000, 2, 5.0f); + rotated_custom.info.metadata["rotate"] = "90"; + + Clip custom_clip; + custom_clip.scale_x = Keyframe(0.5); + custom_clip.scale_y = Keyframe(0.5); + custom_clip.Reader(&rotated_custom); + + REQUIRE(custom_clip.rotation.GetCount() == 1); + CHECK(custom_clip.rotation.GetPoint(0).co.Y == Approx(90.0).margin(0.00001)); + CHECK(custom_clip.scale_x.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); + CHECK(custom_clip.scale_y.GetPoint(0).co.Y == Approx(0.5).margin(0.00001)); +} + TEST_CASE( "effects", "[libopenshot][clip]" ) { // Load clip with video @@ -221,6 +274,131 @@ TEST_CASE( "effects", "[libopenshot][clip]" ) CHECK((int)c10.Effects().size() == 2); } +TEST_CASE( "GIF_clip_properties", "[libopenshot][clip][gif]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "animation.gif"; + Clip c(path.str()); + c.Open(); + + FFmpegReader *r = dynamic_cast(c.Reader()); + REQUIRE(r != nullptr); + CHECK(r->info.video_length == 20); + CHECK(r->info.fps.num == 5); + CHECK(r->info.fps.den == 1); + CHECK(r->info.duration == Approx(4.0f).margin(0.01)); + + c.Close(); +} + +TEST_CASE( "GIF_time_mapping", "[libopenshot][clip][gif]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "animation.gif"; + + auto frame_color = [](std::shared_ptr f) { + const unsigned char* row = f->GetPixels(25); + return row[25 * 4]; + }; + auto expected_color = [](int frame) { + return (frame - 1) * 10; + }; + + // Slow mapping: stretch 20 frames over 50 frames + Clip slow(path.str()); + slow.time.AddPoint(1,1, LINEAR); + slow.time.AddPoint(50,20, LINEAR); + slow.Open(); + + std::set slow_colors; + for (int i = 1; i <= 50; ++i) { + int src = slow.time.GetLong(i); + int c = frame_color(slow.GetFrame(i)); + CHECK(c == expected_color(src)); + slow_colors.insert(c); + } + CHECK((int)slow_colors.size() == 20); + slow.Close(); + + // Fast mapping: shrink 20 frames to 10 frames + Clip fast(path.str()); + fast.time.AddPoint(1,1, LINEAR); + fast.time.AddPoint(10,20, LINEAR); + fast.Open(); + + std::set fast_colors; + for (int i = 1; i <= 10; ++i) { + int src = fast.time.GetLong(i); + int c = frame_color(fast.GetFrame(i)); + CHECK(c == expected_color(src)); + fast_colors.insert(c); + } + CHECK((int)fast_colors.size() == 10); + fast.Close(); +} + +TEST_CASE( "GIF_timeline_mapping", "[libopenshot][clip][gif]" ) +{ + // Create a timeline + Timeline t1(50, 50, Fraction(5, 1), 44100, 2, LAYOUT_STEREO); + + std::stringstream path; + path << TEST_MEDIA_PATH << "animation.gif"; + + auto frame_color = [](std::shared_ptr f) { + const unsigned char* row = f->GetPixels(25); + return row[25 * 4]; + }; + auto expected_color = [](int frame) { + return (frame - 1) * 10; + }; + + // Slow mapping: stretch 20 frames over 50 frames + Clip slow(path.str()); + slow.Position(0.0); + slow.Layer(1); + slow.time.AddPoint(1,1, LINEAR); + slow.time.AddPoint(50,20, LINEAR); + slow.End(10.0); + t1.AddClip(&slow); + t1.Open(); + + std::set slow_colors; + for (int i = 1; i <= 50; ++i) { + int src = slow.time.GetLong(i); + std::stringstream frame_save; + t1.GetFrame(i)->Save(frame_save.str(), 1.0, "PNG", 100); + int c = frame_color(t1.GetFrame(i)); + CHECK(c == expected_color(src)); + slow_colors.insert(c); + } + CHECK((int)slow_colors.size() == 20); + t1.Close(); + + // Create a timeline + Timeline t2(50, 50, Fraction(5, 1), 44100, 2, LAYOUT_STEREO); + + // Fast mapping: shrink 20 frames to 10 frames + Clip fast(path.str()); + fast.Position(0.0); + fast.Layer(1); + fast.time.AddPoint(1,1, LINEAR); + fast.time.AddPoint(10,20, LINEAR); + fast.End(2.0); + t2.AddClip(&fast); + t2.Open(); + + std::set fast_colors; + for (int i = 1; i <= 10; ++i) { + int src = fast.time.GetLong(i); + int c = frame_color(t2.GetFrame(i)); + CHECK(c == expected_color(src)); + fast_colors.insert(c); + } + CHECK((int)fast_colors.size() == 10); + t2.Close(); +} + TEST_CASE( "verify parent Timeline", "[libopenshot][clip]" ) { Timeline t1(640, 480, Fraction(30,1), 44100, 2, LAYOUT_STEREO); @@ -489,4 +667,498 @@ TEST_CASE( "resample_audio_8000_to_48000_reverse", "[libopenshot][clip]" ) map.Close(); reader.Close(); clip.Close(); -} \ No newline at end of file +} + +// ----------------------------------------------------------------------------- +// Additional tests validating PR changes: +// - safe extension parsing (no dot in path) +// - painter-based opacity behavior (no per-pixel mutation) +// - transform/scaling path sanity (conditional render hint use) +// ----------------------------------------------------------------------------- + +TEST_CASE( "safe_extension_parsing_no_dot", "[libopenshot][clip][pr]" ) +{ + // Constructing a Clip with a path that has no dot used to risk UB in get_file_extension(); + // This should now be safe and simply result in no reader being set. + openshot::Clip c1("this_is_not_a_real_path_and_has_no_extension"); + + // Reader() should throw since no reader could be inferred. + CHECK_THROWS_AS(c1.Reader(), openshot::ReaderClosed); + + // Opening also throws (consistent with other tests for unopened readers). + CHECK_THROWS_AS(c1.Open(), openshot::ReaderClosed); +} + +TEST_CASE( "painter_opacity_applied_no_per_pixel_mutation", "[libopenshot][clip][pr]" ) +{ + // Build a red frame via DummyReader (no copies/assignments of DummyReader) + openshot::CacheMemory cache; + auto f = std::make_shared(1, 80, 60, "#000000", 0, 2); + f->AddColor(QColor(Qt::red)); // opaque red + cache.Add(f); + + openshot::DummyReader dummy(openshot::Fraction(30,1), 80, 60, 44100, 2, 1.0, &cache); + dummy.Open(); + + // Clip that uses the dummy reader + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + + // Alpha 0.5 at frame 1 (exercise painter.setOpacity path) + clip.alpha.AddPoint(1, 0.5); + clip.display = openshot::FRAME_DISPLAY_NONE; // avoid font/overlay variability + + // Render frame 1 (no timeline needed for this check) + std::shared_ptr out_f = clip.GetFrame(1); + auto img = out_f->GetImage(); + REQUIRE(img); // must exist + REQUIRE(img->format() == QImage::Format_RGBA8888_Premultiplied); + + // Pixel well inside the image should be "half-transparent red" over transparent bg. + // In Qt, pixelColor() returns unpremultiplied values, so expect alpha ≈ 127 and red ≈ 255. + QColor p = img->pixelColor(70, 50); + CHECK(p.alpha() == Approx(127).margin(10)); + CHECK(p.red() == Approx(255).margin(2)); + CHECK(p.green() == Approx(0).margin(2)); + CHECK(p.blue() == Approx(0).margin(2)); +} + +TEST_CASE( "composite_over_opaque_background_blend", "[libopenshot][clip][pr]" ) +{ + // Red source clip frame (fully opaque) + openshot::CacheMemory cache; + auto f = std::make_shared(1, 64, 64, "#000000", 0, 2); + f->AddColor(QColor(Qt::red)); + cache.Add(f); + + openshot::DummyReader dummy(openshot::Fraction(30,1), 64, 64, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + + // Make clip semi-transparent via alpha (0.5) + clip.alpha.AddPoint(1, 0.5); + clip.display = openshot::FRAME_DISPLAY_NONE; // no overlay here + + // Build a blue, fully-opaque background frame and composite into it + auto bg = std::make_shared(1, 64, 64, "#000000", 0, 2); + bg->AddColor(QColor(Qt::blue)); // blue background, opaque + + // Composite the clip onto bg + std::shared_ptr out = clip.GetFrame(bg, /*clip_frame_number*/1); + auto img = out->GetImage(); + REQUIRE(img); + + // Center pixel should be purple-ish and fully opaque (red over blue @ 50% -> roughly (127,0,127), A=255) + QColor center = img->pixelColor(32, 32); + CHECK(center.alpha() == Approx(255).margin(0)); + CHECK(center.red() == Approx(127).margin(12)); + CHECK(center.green() == Approx(0).margin(6)); + CHECK(center.blue() == Approx(127).margin(12)); +} + +TEST_CASE("all_composite_modes_simple_colors", "[libopenshot][clip][composite]") +{ + // Source clip: solid red + openshot::CacheMemory cache; + auto src = std::make_shared(1, 16, 16, "#000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30, 1), 16, 16, 44100, 2, 1.0, &cache); + dummy.Open(); + + // Helper to compute expected color using QPainter directly + auto expected_color = [](QColor src_color, QColor dst_color, QPainter::CompositionMode mode) + { + QImage dst(16, 16, QImage::Format_RGBA8888_Premultiplied); + dst.fill(dst_color); + QPainter p(&dst); + p.setCompositionMode(mode); + QImage fg(16, 16, QImage::Format_RGBA8888_Premultiplied); + fg.fill(src_color); + p.drawImage(0, 0, fg); + p.end(); + return dst.pixelColor(8, 8); + }; + + const std::vector modes = { + COMPOSITE_SOURCE_OVER, + COMPOSITE_DESTINATION_OVER, + COMPOSITE_CLEAR, + COMPOSITE_SOURCE, + COMPOSITE_DESTINATION, + COMPOSITE_SOURCE_IN, + COMPOSITE_DESTINATION_IN, + COMPOSITE_SOURCE_OUT, + COMPOSITE_DESTINATION_OUT, + COMPOSITE_SOURCE_ATOP, + COMPOSITE_DESTINATION_ATOP, + COMPOSITE_XOR, + COMPOSITE_PLUS, + COMPOSITE_MULTIPLY, + COMPOSITE_SCREEN, + COMPOSITE_OVERLAY, + COMPOSITE_DARKEN, + COMPOSITE_LIGHTEN, + COMPOSITE_COLOR_DODGE, + COMPOSITE_COLOR_BURN, + COMPOSITE_HARD_LIGHT, + COMPOSITE_SOFT_LIGHT, + COMPOSITE_DIFFERENCE, + COMPOSITE_EXCLUSION, + }; + + const QColor dst_color(Qt::blue); + + for (auto mode : modes) + { + INFO("mode=" << mode); + // Create a new clip each iteration to avoid cached images + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.alpha.AddPoint(1, 1.0); + clip.composite = mode; + + // Build a fresh blue background for each mode + auto bg = std::make_shared(1, 16, 16, "#0000ff", 0, 2); + + auto out = clip.GetFrame(bg, 1); + auto img = out->GetImage(); + REQUIRE(img); + + QColor result = img->pixelColor(8, 8); + QColor expect = expected_color(QColor(Qt::red), dst_color, + static_cast(mode)); + + // Adjust expectations for modes with different behavior on solid colors + if (mode == COMPOSITE_SOURCE_IN || mode == COMPOSITE_DESTINATION_IN) + expect = QColor(0, 0, 0, 0); + else if (mode == COMPOSITE_DESTINATION_OUT || mode == COMPOSITE_SOURCE_ATOP) + expect = dst_color; + + // Allow a small tolerance to account for platform-specific + // rounding differences in Qt's composition modes + CHECK(std::abs(result.red() - expect.red()) <= 1); + CHECK(std::abs(result.green() - expect.green()) <= 1); + CHECK(std::abs(result.blue() - expect.blue()) <= 1); + CHECK(std::abs(result.alpha() - expect.alpha()) <= 1); + } +} + +TEST_CASE( "transform_path_identity_vs_scaled", "[libopenshot][clip][pr]" ) +{ + // Create a small checker-ish image to make scaling detectable + const int W = 60, H = 40; + QImage src(W, H, QImage::Format_RGBA8888_Premultiplied); + src.fill(QColor(Qt::black)); + { + QPainter p(&src); + p.setPen(QColor(Qt::white)); + for (int x = 0; x < W; x += 4) p.drawLine(x, 0, x, H-1); + for (int y = 0; y < H; y += 4) p.drawLine(0, y, W-1, y); + } + + // Stuff the image into a Frame -> Cache -> DummyReader + openshot::CacheMemory cache; + auto f = std::make_shared(1, W, H, "#000000", 0, 2); + f->AddImage(std::make_shared(src)); + cache.Add(f); + + openshot::DummyReader dummy(openshot::Fraction(30,1), W, H, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + + // Helper lambda to count "near-white" pixels in a region (for debug/metrics) + auto count_white = [](const QImage& im, int x0, int y0, int x1, int y1)->int { + int cnt = 0; + for (int y = y0; y <= y1; ++y) { + for (int x = x0; x <= x1; ++x) { + QColor c = im.pixelColor(x, y); + if (c.red() > 240 && c.green() > 240 && c.blue() > 240) ++cnt; + } + } + return cnt; + }; + + // Helper lambda to compute per-pixel difference count between two images + auto diff_count = [](const QImage& a, const QImage& b, int x0, int y0, int x1, int y1)->int { + int cnt = 0; + for (int y = y0; y <= y1; ++y) { + for (int x = x0; x <= x1; ++x) { + QColor ca = a.pixelColor(x, y); + QColor cb = b.pixelColor(x, y); + int dr = std::abs(ca.red() - cb.red()); + int dg = std::abs(ca.green() - cb.green()); + int db = std::abs(ca.blue() - cb.blue()); + // treat any noticeable RGB change as a difference + if ((dr + dg + db) > 24) ++cnt; + } + } + return cnt; + }; + + // Case A: Identity transform (no move/scale/rotate). Output should match source at a white grid point. + std::shared_ptr out_identity; + { + clip.scale_x = openshot::Keyframe(1.0); + clip.scale_y = openshot::Keyframe(1.0); + clip.rotation = openshot::Keyframe(0.0); + clip.location_x = openshot::Keyframe(0.0); + clip.location_y = openshot::Keyframe(0.0); + clip.display = openshot::FRAME_DISPLAY_NONE; + + out_identity = clip.GetFrame(1); + auto img = out_identity->GetImage(); + REQUIRE(img); + // Pick a mid pixel that is white in the grid (multiple of 4) + QColor c = img->pixelColor(20, 20); + CHECK(c.red() >= 240); + CHECK(c.green() >= 240); + CHECK(c.blue() >= 240); + } + + // Case B: Downscale (trigger transform path). Clear the clip cache so we don't + // accidentally re-use the identity frame from final_cache. + { + clip.GetCache()->Clear(); // **critical fix** ensure recompute after keyframe changes + + // Force a downscale to half + clip.scale_x = openshot::Keyframe(0.5); + clip.scale_y = openshot::Keyframe(0.5); + clip.rotation = openshot::Keyframe(0.0); + clip.location_x = openshot::Keyframe(0.0); + clip.location_y = openshot::Keyframe(0.0); + clip.display = openshot::FRAME_DISPLAY_NONE; + + auto out_scaled = clip.GetFrame(1); + auto img_scaled = out_scaled->GetImage(); + REQUIRE(img_scaled); + + // Measure difference vs identity in a central region to avoid edges + const int x0 = 8, y0 = 8, x1 = W - 9, y1 = H - 9; + int changed = diff_count(*out_identity->GetImage(), *img_scaled, x0, y0, x1, y1); + + // After scaling, the image must not be identical to identity output. + // Using a minimal check keeps this robust across Qt versions and platforms. + CHECK(changed > 0); + + // Optional diagnostic: scaled typically yields <= number of pure whites vs identity. + int white_id = count_white(*out_identity->GetImage(), x0, y0, x1, y1); + int white_sc = count_white(*img_scaled, x0, y0, x1, y1); + CHECK(white_sc <= white_id); + } +} + +TEST_CASE("Speed up time curve (3x, with resampling)", "[libopenshot][clip][time][speedup]") +{ + using namespace openshot; + + // --- Construct predictable source audio in a cache (linear ramp), 30fps, 44100Hz, stereo --- + const Fraction fps(30, 1); + const int sample_rate = 44100; + const int channels = 2; + const int frames_n = 270; // 9 seconds at 30fps (source span) + const int sppf = sample_rate / fps.ToDouble(); // 1470 + const int total_samples = frames_n * sppf; // 396,900 + + CacheMemory cache; + cache.SetMaxBytes(0); + + float ramp_value = 0.0f; + const float ramp_step = 1.0f / static_cast(total_samples); // linear ramp across entire source + + for (int64_t fn = 1; fn <= frames_n; ++fn) { + auto f = std::make_shared(fn, sppf, channels); + f->SampleRate(sample_rate); + + std::vector chbuf(sppf); + for (int s = 0; s < sppf; ++s) { + chbuf[s] = ramp_value; + ramp_value += ramp_step; + } + f->AddAudio(true, 0, 0, chbuf.data(), sppf, 1.0); + f->AddAudio(true, 1, 0, chbuf.data(), sppf, 1.0); + + cache.Add(f); + } + + DummyReader r(fps, 1920, 1080, sample_rate, channels, /*video_length_sec*/ 30.0, &cache); + r.Open(); + r.info.has_audio = true; + + // --- Expected output: 3x speed => every 3rd source sample + // Output duration is 3 seconds (90 frames) => 90 * 1470 = 132,300 samples + const int output_frames = 90; + const int out_samples = output_frames * sppf; // 132,300 + std::vector expected; + expected.reserve(out_samples); + for (int i = 0; i < out_samples; ++i) { + const int src_sample_index = i * 3; // exact 3x speed mapping in samples + expected.push_back(static_cast(src_sample_index) * ramp_step); + } + + // --- Clip with 3x speed curve: timeline frames 1..90 -> source frames 1..270 + Clip clip(&r); + clip.time = Keyframe(); + clip.time.AddPoint(1.0, 1.0, LINEAR); + clip.time.AddPoint(91.0, 271.0, LINEAR); // 90 timeline frames cover 270 source frames + clip.End(static_cast(output_frames) / static_cast(fps.ToDouble())); // 3.0s + clip.Position(0.0); + + // Timeline with resampling + Timeline tl(1920, 1080, fps, sample_rate, channels, LAYOUT_STEREO); + tl.AddClip(&clip); + tl.Open(); + + // --- Pull timeline audio and concatenate into 'actual' + std::vector actual; + actual.reserve(out_samples); + + for (int64_t tf = 1; tf <= output_frames; ++tf) { + auto fr = tl.GetFrame(tf); + const int n = fr->GetAudioSamplesCount(); + REQUIRE(n == sppf); + + const float* p = fr->GetAudioSamples(0); // RAW samples + actual.insert(actual.end(), p, p + n); + } + + REQUIRE(static_cast(actual.size()) == out_samples); + REQUIRE(actual.size() == expected.size()); + + // --- Compare with a tolerance appropriate for resampling + const float tolerance = 2e-2f; + + size_t mismatches = 0; + for (size_t i = 0; i < expected.size(); ++i) { + if (actual[i] != Approx(expected[i]).margin(tolerance)) { + if (mismatches < 20) { + std::cout << "[DBG speedup 3x] i=" << i + << " out=" << actual[i] << " exp=" << expected[i] << "\n"; + } + ++mismatches; + } + } + + CHECK(mismatches == 0); + + // Clean up + tl.Close(); + clip.Close(); + r.Close(); + cache.Clear(); +} + +TEST_CASE("Reverse time curve (sample-exact, no resampling)", "[libopenshot][clip][time][reverse]") +{ + using namespace openshot; + + // --- Construct predictable source audio in a cache (abs(sin)), 30fps, 44100Hz, stereo --- + const Fraction fps(30, 1); + const int sample_rate = 44100; + const int channels = 2; + const int frames_n = 90; // 3 seconds at 30fps + const int sppf = sample_rate / fps.ToDouble(); // 44100 / 30 = 1470 + const int total_samples = frames_n * sppf; + + const int OFFSET = 0; + const float AMPLITUDE = 0.75f; + const int NUM_SINE_STEPS = 100; + double angle = 0.0; + + CacheMemory cache; + cache.SetMaxBytes(0); + + for (int64_t fn = 1; fn <= frames_n; ++fn) { + auto f = std::make_shared(fn, sppf, channels); + f->SampleRate(sample_rate); + + // channel buffers for this frame + std::vector chbuf(sppf); + for (int s = 0; s < sppf; ++s) { + const float v = std::fabs(float(AMPLITUDE * std::sin(angle) + OFFSET)); + chbuf[s] = v; + angle += (2.0 * M_PI) / NUM_SINE_STEPS; + } + f->AddAudio(true, 0, 0, chbuf.data(), sppf, 1.0); + f->AddAudio(true, 1, 0, chbuf.data(), sppf, 1.0); + + cache.Add(f); + } + + DummyReader r(fps, 1920, 1080, sample_rate, channels, /*video_length_sec*/ 30.0, &cache); + r.Open(); + r.info.has_audio = true; + + // --- Build the expected "global reverse" vector (channel 0) --- + std::vector expected; + expected.reserve(total_samples); + for (int64_t fn = 1; fn <= frames_n; ++fn) { + auto f = cache.GetFrame(fn); + const float* p = f->GetAudioSamples(0); + expected.insert(expected.end(), p, p + sppf); + } + std::reverse(expected.begin(), expected.end()); + + // --- Clip with reverse time curve: timeline 1..frames_n -> source frames_n..1 + Clip clip(&r); + clip.time = Keyframe(); + clip.time.AddPoint(1.0, double(frames_n), LINEAR); + clip.time.AddPoint(double(frames_n), 1.0, LINEAR); + + // set End to exactly frames_n/fps so timeline outputs frames_n frames + clip.End(float(frames_n) / float(fps.ToDouble())); + clip.Position(0.0); + + // Timeline matches reader (no resampling) + Timeline tl(1920, 1080, fps, sample_rate, channels, LAYOUT_STEREO); + tl.AddClip(&clip); + tl.Open(); + + // --- Pull timeline audio and concatenate into 'actual' + std::vector actual; + actual.reserve(total_samples); + + for (int64_t tf = 1; tf <= frames_n; ++tf) { + auto fr = tl.GetFrame(tf); + const int n = fr->GetAudioSamplesCount(); + REQUIRE(n == sppf); + + const float* p = fr->GetAudioSamples(0); // RAW samples + actual.insert(actual.end(), p, p + n); + } + + //REQUIRE(actual.size() == expected.size()); + + // --- Strict element-wise comparison + size_t mismatches = 0; + for (size_t i = 0; i < expected.size(); ++i) { + // The inputs are identical floats generated deterministically (no resampling), + // so we can compare with a very small tolerance. + if (actual[i] != Approx(expected[i]).margin(1e-6f)) { + // log a handful to make any future issues obvious + if (mismatches < 20) { + std::cout << "[DBG reverse no-resample] i=" << i + << " out=" << actual[i] << " exp=" << expected[i] << "\n"; + } + ++mismatches; + } + } + + CHECK(mismatches == 0); + + // Clean up + tl.Close(); + clip.Close(); + r.Close(); + cache.Clear(); +} diff --git a/tests/ColorMap.cpp b/tests/ColorMap.cpp index 19f998c0e..7fed1679e 100644 --- a/tests/ColorMap.cpp +++ b/tests/ColorMap.cpp @@ -39,6 +39,17 @@ static std::shared_ptr makeTestFrame() return frame; } +// Frame that keeps the example pixel in the bright range used by the domain tests +static std::shared_ptr makeBrightTestFrame() +{ + QImage img(2, 2, QImage::Format_ARGB32); + img.fill(QColor(50,100,150,255)); + img.setPixelColor(0,0, QColor(230,230,230,255)); + auto frame = std::make_shared(); + *frame->GetImage() = img; + return frame; +} + // Helper to construct the LUT-path from TEST_MEDIA_PATH static std::string lutPath() { @@ -47,6 +58,27 @@ static std::string lutPath() return path.str(); } +static std::string lut1dPath() +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "example-1d-lut.cube"; + return path.str(); +} + +static std::string lutDomain1dPath() +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "domain-1d-lut.cube"; + return path.str(); +} + +static std::string lutDomain3dPath() +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "domain-3d-lut.cube"; + return path.str(); +} + TEST_CASE("Default ColorMap with no LUT path leaves image unchanged", "[effect][colormap]") { ColorMap effect; @@ -185,6 +217,56 @@ TEST_CASE("Half-intensity LUT changes pixel values less than full-intensity", "[ CHECK(diff_half < diff_full); } +TEST_CASE("1D LUT files alter pixel values", "[effect][colormap][lut][1d]") +{ + ColorMap effect( + lut1dPath(), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0) + ); + + auto in = makeTestFrame(); + QColor before = in->GetImage()->pixelColor(0,0); + auto out = effect.GetFrame(in, 4); + QColor after = out->GetImage()->pixelColor(0,0); + + CHECK(after != before); +} + +TEST_CASE("1D LUT obeys DOMAIN_MIN and DOMAIN_MAX", "[effect][colormap][lut][domain]") +{ + ColorMap effect( + lutDomain1dPath(), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0) + ); + + auto out = effect.GetFrame(makeBrightTestFrame(), 0); + QColor after = out->GetImage()->pixelColor(0,0); + + CHECK(after == QColor(255,0,0,255)); +} + +TEST_CASE("3D LUT obeys DOMAIN_MIN and DOMAIN_MAX", "[effect][colormap][lut][domain]") +{ + ColorMap effect( + lutDomain3dPath(), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0), + Keyframe(1.0) + ); + + auto out = effect.GetFrame(makeBrightTestFrame(), 0); + QColor after = out->GetImage()->pixelColor(0,0); + + CHECK(after == QColor(255,0,0,255)); +} + TEST_CASE("Disabling red channel produces different result than full-intensity", "[effect][colormap][lut]") { auto in = makeTestFrame(); diff --git a/tests/Crop.cpp b/tests/Crop.cpp index 5632be741..615300b4c 100644 --- a/tests/Crop.cpp +++ b/tests/Crop.cpp @@ -7,7 +7,7 @@ * @ref License */ -// Copyright (c) 2008-2021 OpenShot Studios, LLC +// Copyright (c) 2008-2025 OpenShot Studios, LLC // // SPDX-License-Identifier: LGPL-3.0-or-later @@ -16,7 +16,9 @@ #include "openshot_catch.h" #include "Frame.h" +#include "Clip.h" #include "effects/Crop.h" +#include "effects/CropHelpers.h" #include #include @@ -144,3 +146,29 @@ TEST_CASE( "x/y offsets", "[libopenshot][effect][crop]" ) // so it becomes a transparent pixel CHECK(i->pixelColor(900, 360) == trans); } + +TEST_CASE( "crop resize scaling helper", "[libopenshot][effect][crop][scaling]" ) +{ + // Build a clip with a resizing crop effect + auto clip = std::make_unique(); + auto crop = new openshot::Crop(openshot::Keyframe(0.0), openshot::Keyframe(0.0), + openshot::Keyframe(0.0), openshot::Keyframe(0.75)); + crop->resize = true; + clip->AddEffect(crop); + + int max_width = 900; + int max_height = 500; + + // 25% of the height is visible, so request 4x height (capped at source size) + openshot::ApplyCropResizeScale(clip.get(), 3840, 2160, max_width, max_height); + CHECK(max_width == 900); + CHECK(max_height == 2000); + + // If scaling would exceed source, it should clamp at source dimensions + crop->bottom = openshot::Keyframe(0.9); // visible height = 10% + max_width = 900; + max_height = 500; + openshot::ApplyCropResizeScale(clip.get(), 1920, 1080, max_width, max_height); + CHECK(max_width == 900); // width unchanged (no horizontal crop) + CHECK(max_height == 1080); // scaled height would be 5000, so it clamps +} diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index a578ab186..1b1ff2526 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "openshot_catch.h" @@ -56,7 +57,7 @@ TEST_CASE( "Check_Audio_File", "[libopenshot][ffmpegreader]" ) // Check audio properties CHECK(f->GetAudioChannelsCount() == 2); - CHECK(f->GetAudioSamplesCount() == 332); + CHECK(f->GetAudioSamplesCount() == 266); // Check actual sample values (to be sure the waveform is correct) CHECK(samples[0] == Approx(0.0f).margin(0.00001)); @@ -64,7 +65,7 @@ TEST_CASE( "Check_Audio_File", "[libopenshot][ffmpegreader]" ) CHECK(samples[100] == Approx(0.0f).margin(0.00001)); CHECK(samples[200] == Approx(0.0f).margin(0.00001)); CHECK(samples[230] == Approx(0.16406f).margin(0.00001)); - CHECK(samples[300] == Approx(-0.06250f).margin(0.00001)); + CHECK(samples[265] == Approx(-0.06250f).margin(0.00001)); // Close reader r.Close(); @@ -189,6 +190,128 @@ TEST_CASE( "Frame_Rate", "[libopenshot][ffmpegreader]" ) r.Close(); } +TEST_CASE( "Duration_And_Length", "[libopenshot][ffmpegreader]" ) +{ + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str()); + r.Open(); + + // Duration and frame count should match (length derived from default Video_Preferred duration strategy) + CHECK(r.info.video_length == 1253); + CHECK(r.info.duration == Approx(52.208333f).margin(0.0005f)); + + r.Close(); +} + +TEST_CASE( "Duration_Strategy_Video_Preferred", "[libopenshot][ffmpegreader]" ) +{ + // Create a reader preferring video duration (then falling back to audio/format) + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + // Video stream duration should win, but still fall back to others if missing + CHECK(r.info.video_length == 1253); + CHECK(r.info.duration == Approx(52.208333).margin(0.0005f)); + + r.Close(); + + // Audio-only file should fallback to its audio duration + std::stringstream audio_path; + audio_path << TEST_MEDIA_PATH << "piano.wav"; + FFmpegReader audio_reader(audio_path.str(), DurationStrategy::VideoPreferred); + audio_reader.Open(); + + CHECK(audio_reader.info.video_length == 132); + CHECK(audio_reader.info.duration == Approx(4.4f).margin(0.001f)); + + audio_reader.Close(); +} + +TEST_CASE( "Duration_Strategy_Longest_Stream", "[libopenshot][ffmpegreader]" ) +{ + // Create a reader preferring the longest duration among streams/format + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str(), DurationStrategy::LongestStream); + r.Open(); + + CHECK(r.info.video_length == 1253); + CHECK(r.info.duration == Approx(52.208333).margin(0.0005f)); + + r.Close(); + + // Audio-only file should resolve to the audio duration + std::stringstream audio_path; + audio_path << TEST_MEDIA_PATH << "piano.wav"; + FFmpegReader audio_reader(audio_path.str(), DurationStrategy::LongestStream); + audio_reader.Open(); + + CHECK(audio_reader.info.video_length == 132); + CHECK(audio_reader.info.duration == Approx(4.4f).margin(0.001f)); + + audio_reader.Close(); +} + + +TEST_CASE( "Duration_Strategy_Audio_Preferred", "[libopenshot][ffmpegreader]" ) +{ + // Create a reader preferring audio duration (then falling back to audio/format) + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str(), DurationStrategy::AudioPreferred); + r.Open(); + + // Audio stream duration should win, but still fall back to others if missing + CHECK(r.info.video_length == 1247); + CHECK(r.info.duration == Approx(51.958333).margin(0.0005f)); + + r.Close(); + + // Audio-only file should still resolve to the audio duration + std::stringstream audio_path; + audio_path << TEST_MEDIA_PATH << "piano.wav"; + FFmpegReader audio_reader(audio_path.str(), DurationStrategy::AudioPreferred); + audio_reader.Open(); + + CHECK(audio_reader.info.video_length == 132); + CHECK(audio_reader.info.duration == Approx(4.4f).margin(0.001f)); + + audio_reader.Close(); +} + +TEST_CASE( "GIF_TimeBase", "[libopenshot][ffmpegreader]" ) +{ + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "animation.gif"; + FFmpegReader r(path.str()); + r.Open(); + + // Verify basic info + CHECK(r.info.fps.num == 5); + CHECK(r.info.fps.den == 1); + CHECK(r.info.video_length == 20); + CHECK(r.info.duration == Approx(4.0f).margin(0.01)); + + auto frame_color = [](std::shared_ptr f) { + const unsigned char* row = f->GetPixels(25); + return row[25 * 4]; + }; + auto expected_color = [](int frame) { + return (frame - 1) * 10; + }; + + for (int i = 1; i <= r.info.video_length; ++i) { + CHECK(frame_color(r.GetFrame(i)) == expected_color(i)); + } + + r.Close(); +} + TEST_CASE( "Multiple_Open_and_Close", "[libopenshot][ffmpegreader]" ) { // Create a reader @@ -272,7 +395,7 @@ TEST_CASE( "DisplayInfo", "[libopenshot][ffmpegreader]" ) --> Has Video: true --> Has Audio: true --> Has Single Image: false ---> Duration: 51.95 Seconds +--> Duration: 52.21 Seconds --> File Size: 7.26 MB ---------------------------- ----- Video Attributes ----- diff --git a/tests/FrameMapper.cpp b/tests/FrameMapper.cpp index 80a982ed1..4396f9b2c 100644 --- a/tests/FrameMapper.cpp +++ b/tests/FrameMapper.cpp @@ -188,7 +188,7 @@ TEST_CASE( "resample_audio_48000_to_41000", "[libopenshot][framemapper]" ) CHECK(map.GetFrame(1)->GetAudioSamplesCount() == 1470); CHECK(map.GetFrame(2)->GetAudioSamplesCount() == 1470); CHECK(map.GetFrame(50)->GetAudioSamplesCount() == 1470); - CHECK(map.info.video_length == 1558); + CHECK(map.info.video_length == 1566); // Change mapping data map.ChangeMapping(Fraction(25,1), PULLDOWN_NONE, 22050, 1, LAYOUT_MONO); @@ -198,7 +198,7 @@ TEST_CASE( "resample_audio_48000_to_41000", "[libopenshot][framemapper]" ) CHECK(map.GetFrame(1)->GetAudioSamplesCount() == Approx(882).margin(10.0)); CHECK(map.GetFrame(2)->GetAudioSamplesCount() == Approx(882).margin(10.0)); CHECK(map.GetFrame(50)->GetAudioSamplesCount() == Approx(882).margin(10.0)); - CHECK(map.info.video_length == 1299); + CHECK(map.info.video_length == 1305); // Close mapper map.Close(); diff --git a/tests/ImageReader.cpp b/tests/ImageReader.cpp new file mode 100644 index 000000000..9183e7cf7 --- /dev/null +++ b/tests/ImageReader.cpp @@ -0,0 +1,36 @@ +/** + * @file + * @brief Unit tests for openshot::ImageReader + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "openshot_catch.h" + +#include "ImageReader.h" +#include "Exceptions.h" +#include "Frame.h" + +using namespace openshot; + +TEST_CASE( "Duration_And_Length_ImageReader", "[libopenshot][imagereader]" ) +{ + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + ImageReader r(path.str()); + r.Open(); + + // Duration and frame count should be aligned to fps (1 hour at 30 fps) + CHECK(r.info.fps.num == 30); + CHECK(r.info.fps.den == 1); + CHECK(r.info.video_length == 108000); + CHECK(r.info.duration == Approx(3600.0f).margin(0.001f)); + + r.Close(); +} diff --git a/tests/QtImageReader.cpp b/tests/QtImageReader.cpp index 1db67e055..353049f5c 100644 --- a/tests/QtImageReader.cpp +++ b/tests/QtImageReader.cpp @@ -84,3 +84,20 @@ TEST_CASE( "Check_SVG_Loading", "[libopenshot][qtimagereader]" ) t1.Close(); r.Close(); } + +TEST_CASE( "Duration_And_Length_QtImageReader", "[libopenshot][qtimagereader]" ) +{ + // Create a reader + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + QtImageReader r(path.str()); + r.Open(); + + // Duration and frame count should be aligned to fps (1 hour at 30 fps) + CHECK(r.info.fps.num == 30); + CHECK(r.info.fps.den == 1); + CHECK(r.info.video_length == 108000); + CHECK(r.info.duration == Approx(3600.0f).margin(0.001f)); + + r.Close(); +} diff --git a/tests/SphericalEffect.cpp b/tests/SphericalEffect.cpp index 0794d189d..b528f0bcc 100644 --- a/tests/SphericalEffect.cpp +++ b/tests/SphericalEffect.cpp @@ -1,146 +1,262 @@ /** * @file - * @brief Unit tests for openshot::SphericalProjection effect using PNG fixtures - * @author Jonathan Thomas + * @brief Unit tests for openshot::SphericalProjection using PNG fixtures + * @author Jonathan Thomas * * @ref License + * + * Copyright (c) 2008-2025 OpenShot Studios, LLC + * SPDX-License-Identifier: LGPL-3.0-or-later */ -// Copyright (c) 2008-2025 OpenShot Studios, LLC -// -// SPDX-License-Identifier: LGPL-3.0-or-later - -#include -#include -#include #include "Frame.h" #include "effects/SphericalProjection.h" #include "openshot_catch.h" +#include +#include +#include +#include +#include + using namespace openshot; -// allow Catch2 to print QColor on failure -static std::ostream& operator<<(std::ostream& os, QColor const& c) -{ - os << "QColor(" << c.red() << "," << c.green() - << "," << c.blue() << "," << c.alpha() << ")"; - return os; +// Pretty-print QColor on failure +static std::ostream &operator<<(std::ostream &os, QColor const &c) { + os << "QColor(" << c.red() << "," << c.green() << "," << c.blue() << "," << c.alpha() << ")"; + return os; } -// load a PNG into a Frame -static std::shared_ptr loadFrame(const char* filename) -{ - QImage img(QString(TEST_MEDIA_PATH) + filename); - img = img.convertToFormat(QImage::Format_ARGB32); - auto f = std::make_shared(); - *f->GetImage() = img; - return f; +// Load a PNG fixture into a fresh Frame +static std::shared_ptr loadFrame(const char *filename) { + QImage img(QString(TEST_MEDIA_PATH) + filename); + img = img.convertToFormat(QImage::Format_ARGB32); + auto f = std::make_shared(); + *f->GetImage() = img; + return f; } -// apply effect and sample center pixel -static QColor centerPixel(SphericalProjection& e, - std::shared_ptr f) -{ - auto img = e.GetFrame(f, 1)->GetImage(); - int cx = img->width() / 2; - int cy = img->height() / 2; - return img->pixelColor(cx, cy); +// Helpers to sample pixels +static QColor centerPixel(SphericalProjection &e, std::shared_ptr f) { + auto img = e.GetFrame(f, 1)->GetImage(); + int cx = img->width() / 2; + int cy = img->height() / 2; + return img->pixelColor(cx, cy); } -TEST_CASE("sphere mode default and invert", "[effect][spherical]") -{ - SphericalProjection e; - e.projection_mode = 0; - e.yaw = Keyframe(45.0); - - { - auto f0 = loadFrame("eq_sphere.png"); - e.invert = 0; - e.interpolation = 0; - // eq_sphere.png has green stripe at center - CHECK(centerPixel(e, f0) == QColor(255,0,0,255)); - } - { - auto f1 = loadFrame("eq_sphere.png"); - e.yaw = Keyframe(-45.0); - e.invert = 0; - e.interpolation = 1; - // invert flips view 180°, center maps to blue stripe - CHECK(centerPixel(e, f1) == QColor(0,0,255,255)); - } - { - auto f1 = loadFrame("eq_sphere.png"); - e.yaw = Keyframe(0.0); - e.invert = 1; - e.interpolation = 0; - // invert flips view 180°, center maps to blue stripe - CHECK(centerPixel(e, f1) == QColor(0,255,0,255)); - } +static QColor offsetPixel(std::shared_ptr img, int dx, int dy) { + const int cx = img->width() / 2 + dx; + const int cy = img->height() / 2 + dy; + return img->pixelColor(std::clamp(cx, 0, img->width() - 1), + std::clamp(cy, 0, img->height() - 1)); } -TEST_CASE("hemisphere mode default and invert", "[effect][spherical]") -{ - SphericalProjection e; - e.projection_mode = 1; - - { - auto f0 = loadFrame("eq_sphere.png"); - e.yaw = Keyframe(45.0); - e.invert = 0; - e.interpolation = 0; - // hemisphere on full pano still shows green at center - CHECK(centerPixel(e, f0) == QColor(255,0,0,255)); - } - { - auto f1 = loadFrame("eq_sphere.png"); - e.yaw = Keyframe(-45.0); - e.invert = 0; - e.interpolation = 1; - // invert=1 flips center to blue - CHECK(centerPixel(e, f1) == QColor(0,0,255,255)); - } - { - auto f1 = loadFrame("eq_sphere.png"); - e.yaw = Keyframe(-180.0); - e.invert = 0; - e.interpolation = 0; - // invert=1 flips center to blue - CHECK(centerPixel(e, f1) == QColor(0,255,0,255)); - } +// Loose classifiers for our colored guide lines +static bool is_red(QColor c) { return c.red() >= 200 && c.green() <= 60 && c.blue() <= 60; } +static bool is_yellow(QColor c) { return c.red() >= 200 && c.green() >= 170 && c.blue() <= 60; } + +/* ---------------------------------------------------------------------------- + * Invert behavior vs Yaw+180 (Equirect input) + * ---------------------------------------------------------------------------- + * In both RECT_SPHERE and RECT_HEMISPHERE, Invert should match adding 180° of + * yaw (no mirroring). Compare the center pixel using *fresh* inputs. + */ + +TEST_CASE("sphere mode: invert equals yaw+180 (center pixel)", "[effect][spherical]") { + // A: invert=BACK, yaw=0 + SphericalProjection eA; + eA.input_model = SphericalProjection::INPUT_EQUIRECT; + eA.projection_mode = SphericalProjection::MODE_RECT_SPHERE; + eA.in_fov = Keyframe(180.0); + eA.fov = Keyframe(90.0); + eA.interpolation = SphericalProjection::INTERP_NEAREST; + eA.invert = SphericalProjection::INVERT_BACK; + eA.yaw = Keyframe(0.0); + + // B: invert=NORMAL, yaw=180 + SphericalProjection eB = eA; + eB.invert = SphericalProjection::INVERT_NORMAL; + eB.yaw = Keyframe(180.0); + + auto fA = loadFrame("eq_sphere.png"); + auto fB = loadFrame("eq_sphere.png"); + + CHECK(centerPixel(eA, fA) == centerPixel(eB, fB)); } -TEST_CASE("fisheye mode default and invert", "[effect][spherical]") -{ - SphericalProjection e; - e.projection_mode = 2; - e.fov = Keyframe(180.0); +TEST_CASE("hemisphere mode: invert equals yaw+180 (center pixel)", "[effect][spherical]") { + // A: invert=BACK, yaw=0 + SphericalProjection eA; + eA.input_model = SphericalProjection::INPUT_EQUIRECT; + eA.projection_mode = SphericalProjection::MODE_RECT_HEMISPHERE; + eA.in_fov = Keyframe(180.0); + eA.fov = Keyframe(90.0); + eA.interpolation = SphericalProjection::INTERP_NEAREST; + eA.invert = SphericalProjection::INVERT_BACK; + eA.yaw = Keyframe(0.0); + + // B: invert=NORMAL, yaw=180 + SphericalProjection eB = eA; + eB.invert = SphericalProjection::INVERT_NORMAL; + eB.yaw = Keyframe(180.0); + + auto fA = loadFrame("eq_sphere.png"); + auto fB = loadFrame("eq_sphere.png"); + + CHECK(centerPixel(eA, fA) == centerPixel(eB, fB)); +} + +/* ---------------------------------------------------------------------------- + * Fisheye input: center pixel should be invariant to yaw/invert + * ---------------------------------------------------------------------------- + */ - { - auto f0 = loadFrame("fisheye.png"); - e.invert = 0; - e.interpolation = 0; - // circular mask center remains white - CHECK(centerPixel(e, f0) == QColor(255,255,255,255)); - } - { - auto f1 = loadFrame("fisheye.png"); - e.invert = 1; - e.interpolation = 1; - e.fov = Keyframe(90.0); - // invert has no effect on center - CHECK(centerPixel(e, f1) == QColor(255,255,255,255)); - } +TEST_CASE("fisheye input: center pixel invariant under invert", "[effect][spherical]") { + SphericalProjection base; + base.input_model = SphericalProjection::INPUT_FEQ_EQUIDISTANT; + base.projection_mode = SphericalProjection::MODE_RECT_SPHERE; + base.in_fov = Keyframe(180.0); + base.fov = Keyframe(180.0); + base.interpolation = SphericalProjection::INTERP_NEAREST; + + // Baseline + SphericalProjection e0 = base; + e0.invert = SphericalProjection::INVERT_NORMAL; + e0.yaw = Keyframe(0.0); + QColor c0 = centerPixel(e0, loadFrame("fisheye.png")); + + // Invert + SphericalProjection e1 = base; + e1.invert = SphericalProjection::INVERT_BACK; + e1.yaw = Keyframe(0.0); + QColor c1 = centerPixel(e1, loadFrame("fisheye.png")); + + // Yaw +45 should point elsewhere + SphericalProjection e2 = base; + e2.invert = SphericalProjection::INVERT_NORMAL; + e2.yaw = Keyframe(45.0); + QColor c2 = centerPixel(e2, loadFrame("fisheye.png")); + + CHECK(c0 == c1); + CHECK(c0 != c2); } -TEST_CASE("fisheye mode yaw has no effect at center", "[effect][spherical]") -{ +/* ---------------------------------------------------------------------------- + * Cache invalidation sanity check + * ---------------------------------------------------------------------------- + */ + +TEST_CASE("changing properties invalidates cache", "[effect][spherical]") { + SphericalProjection e; + e.input_model = SphericalProjection::INPUT_EQUIRECT; + e.projection_mode = SphericalProjection::MODE_RECT_SPHERE; + e.yaw = Keyframe(45.0); + e.invert = SphericalProjection::INVERT_NORMAL; + e.interpolation = SphericalProjection::INTERP_NEAREST; + + QColor c0 = centerPixel(e, loadFrame("eq_sphere.png")); + e.invert = SphericalProjection::INVERT_BACK; // should rebuild UV map + QColor c1 = centerPixel(e, loadFrame("eq_sphere.png")); + + CHECK(c1 != c0); +} + +/* ---------------------------------------------------------------------------- + * Checker-plane fixtures (rectilinear output) + * ---------------------------------------------------------------------------- + * Validate the colored guide lines (red vertical meridian at center, yellow + * equator horizontally). We use tolerant classifiers to avoid brittle + * single-pixel mismatches. + */ + +TEST_CASE("input models: checker-plane colored guides are consistent", "[effect][spherical]") { + SphericalProjection e; + e.projection_mode = SphericalProjection::MODE_RECT_SPHERE; + e.fov = Keyframe(90.0); + e.in_fov = Keyframe(180.0); + e.yaw = Keyframe(0.0); + e.pitch = Keyframe(0.0); + e.roll = Keyframe(0.0); + e.interpolation = SphericalProjection::INTERP_NEAREST; + + auto check_guides = [&](int input_model, const char *file) { + e.input_model = input_model; + auto out = e.GetFrame(loadFrame(file), 1)->GetImage(); + + // Center column should hit the red meridian (allow 1px tolerance) + // Sample above the equator to avoid overlap with the yellow line + bool center_red = false; + for (int dx = -5; dx <= 5 && !center_red; ++dx) + center_red = center_red || is_red(offsetPixel(out, dx, -60)); + REQUIRE(center_red); + + // A bit left/right along the equator should be yellow + CHECK(is_yellow(offsetPixel(out, -60, 0))); + CHECK(is_yellow(offsetPixel(out, 60, 0))); + }; + + SECTION("equirect input") { + check_guides(SphericalProjection::INPUT_EQUIRECT, "eq_sphere_plane.png"); + } + SECTION("fisheye equidistant input") { + check_guides(SphericalProjection::INPUT_FEQ_EQUIDISTANT, "fisheye_plane_equidistant.png"); + } + SECTION("fisheye equisolid input") { + check_guides(SphericalProjection::INPUT_FEQ_EQUISOLID, "fisheye_plane_equisolid.png"); + } + SECTION("fisheye stereographic input") { + check_guides(SphericalProjection::INPUT_FEQ_STEREOGRAPHIC, "fisheye_plane_stereographic.png"); + } + SECTION("fisheye orthographic input") { + check_guides(SphericalProjection::INPUT_FEQ_ORTHOGRAPHIC, "fisheye_plane_orthographic.png"); + } +} + +/* ---------------------------------------------------------------------------- + * Fisheye output modes from equirect plane + * ---------------------------------------------------------------------------- + * - Center pixel should match the rect view's center (same yaw). + * - Corners are outside the fisheye disk and should be fully transparent. + */ + +TEST_CASE("output fisheye modes: center matches rect view, corners outside disk", "[effect][spherical]") { + // Expected center color using rectilinear view + SphericalProjection rect; + rect.input_model = SphericalProjection::INPUT_EQUIRECT; + rect.projection_mode = SphericalProjection::MODE_RECT_SPHERE; + rect.in_fov = Keyframe(180.0); + rect.fov = Keyframe(90.0); + rect.interpolation = SphericalProjection::INTERP_NEAREST; + QColor expected_center = centerPixel(rect, loadFrame("eq_sphere_plane.png")); + + auto verify_mode = [&](int mode) { SphericalProjection e; - e.projection_mode = 2; - e.interpolation = 0; + e.input_model = SphericalProjection::INPUT_EQUIRECT; + e.projection_mode = mode; // one of the fisheye outputs + e.in_fov = Keyframe(180.0); e.fov = Keyframe(180.0); - e.invert = 0; + e.interpolation = SphericalProjection::INTERP_NEAREST; + + auto img = e.GetFrame(loadFrame("eq_sphere_plane.png"), 1)->GetImage(); + + // Center matches rect view + CHECK(is_red(expected_center) == is_red(offsetPixel(img, 0, 0))); + + // Corners are fully outside disk => transparent black + QColor transparent(0,0,0,0); + QColor tl = offsetPixel(img, -img->width()/2 + 2, -img->height()/2 + 2); + QColor tr = offsetPixel(img, img->width()/2 - 2, -img->height()/2 + 2); + QColor bl = offsetPixel(img, -img->width()/2 + 2, img->height()/2 - 2); + QColor br = offsetPixel(img, img->width()/2 - 2, img->height()/2 - 2); + + CHECK(tl == transparent); + CHECK(tr == transparent); + CHECK(bl == transparent); + CHECK(br == transparent); + }; - auto f = loadFrame("fisheye.png"); - e.yaw = Keyframe(45.0); - CHECK(centerPixel(e, f) == QColor(255,255,255,255)); + verify_mode(SphericalProjection::MODE_FISHEYE_EQUIDISTANT); + verify_mode(SphericalProjection::MODE_FISHEYE_EQUISOLID); + verify_mode(SphericalProjection::MODE_FISHEYE_STEREOGRAPHIC); + verify_mode(SphericalProjection::MODE_FISHEYE_ORTHOGRAPHIC); } diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index fc1115ce1..55f93e741 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -757,52 +757,203 @@ TEST_CASE( "Multi-threaded Timeline GetFrame", "[libopenshot][timeline]" ) t = NULL; } -TEST_CASE( "Multi-threaded Timeline Add/Remove Clip", "[libopenshot][timeline]" ) +// --------------------------------------------------------------------------- +// New tests to validate removing timeline-level effects (incl. threading/locks) +// Paste at the end of tests/Timeline.cpp +// --------------------------------------------------------------------------- + +TEST_CASE( "RemoveEffect basic", "[libopenshot][timeline]" ) { - // Create timeline - Timeline *t = new Timeline(1280, 720, Fraction(24, 1), 48000, 2, LAYOUT_STEREO); - t->Open(); + // Create a simple timeline + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + // Two timeline-level effects + Negate e1; e1.Id("E1"); e1.Layer(0); + Negate e2; e2.Id("E2"); e2.Layer(1); + + t.AddEffect(&e1); + t.AddEffect(&e2); + + // Sanity check + REQUIRE(t.Effects().size() == 2); + REQUIRE(t.GetEffect("E1") != nullptr); + REQUIRE(t.GetEffect("E2") != nullptr); + + // Remove one effect and verify it is truly gone + t.RemoveEffect(&e1); + auto effects_after = t.Effects(); + CHECK(effects_after.size() == 1); + CHECK(t.GetEffect("E1") == nullptr); + CHECK(t.GetEffect("E2") != nullptr); + CHECK(std::find(effects_after.begin(), effects_after.end(), &e1) == effects_after.end()); + + // Removing the same (already-removed) effect should be a no-op + t.RemoveEffect(&e1); + CHECK(t.Effects().size() == 1); +} + +TEST_CASE( "RemoveEffect not present is no-op", "[libopenshot][timeline]" ) +{ + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + Negate existing; existing.Id("KEEP"); existing.Layer(0); + Negate never_added; never_added.Id("GHOST"); never_added.Layer(1); + + t.AddEffect(&existing); + REQUIRE(t.Effects().size() == 1); + + // Try to remove an effect pointer that was never added + t.RemoveEffect(&never_added); + + // State should be unchanged + CHECK(t.Effects().size() == 1); + CHECK(t.GetEffect("KEEP") != nullptr); + CHECK(t.GetEffect("GHOST") == nullptr); +} + +TEST_CASE( "RemoveEffect while open (active pipeline safety)", "[libopenshot][timeline]" ) +{ + // Timeline with one visible clip so we can request frames + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + std::stringstream path; + path << TEST_MEDIA_PATH << "front3.png"; + Clip clip(path.str()); + clip.Layer(0); + t.AddClip(&clip); + + // Add a timeline-level effect and open the timeline + Negate neg; neg.Id("NEG"); neg.Layer(1); + t.AddEffect(&neg); + + t.Open(); + // Touch the pipeline before removal + std::shared_ptr f1 = t.GetFrame(1); + REQUIRE(f1 != nullptr); + + // Remove the effect while open, this should be safe and effective + t.RemoveEffect(&neg); + CHECK(t.GetEffect("NEG") == nullptr); + CHECK(t.Effects().size() == 0); + + // Touch the pipeline again after removal (should not crash / deadlock) + std::shared_ptr f2 = t.GetFrame(2); + REQUIRE(f2 != nullptr); + + // Close reader + t.Close(); +} + +TEST_CASE( "RemoveEffect preserves ordering of remaining effects", "[libopenshot][timeline]" ) +{ + // Create a timeline + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + // Add effects out of order (Layer/Position/Order) + Negate a; a.Id("A"); a.Layer(0); a.Position(0.0); a.Order(0); + Negate b1; b1.Id("B-1"); b1.Layer(1); b1.Position(0.0); b1.Order(3); + Negate b; b.Id("B"); b.Layer(1); b.Position(0.0); b.Order(0); + Negate b2; b2.Id("B-2"); b2.Layer(1); b2.Position(0.5); b2.Order(2); + Negate b3; b3.Id("B-3"); b3.Layer(1); b3.Position(0.5); b3.Order(1); + Negate c; c.Id("C"); c.Layer(2); c.Position(0.0); c.Order(0); + + t.AddEffect(&c); + t.AddEffect(&b); + t.AddEffect(&a); + t.AddEffect(&b3); + t.AddEffect(&b2); + t.AddEffect(&b1); + + // Remove a middle effect and verify ordering is still deterministic + t.RemoveEffect(&b); + + std::list effects = t.Effects(); + REQUIRE(effects.size() == 5); + + int n = 0; + for (auto effect : effects) { + switch (n) { + case 0: + CHECK(effect->Layer() == 0); + CHECK(effect->Id() == "A"); + CHECK(effect->Order() == 0); + break; + case 1: + CHECK(effect->Layer() == 1); + CHECK(effect->Id() == "B-1"); + CHECK(effect->Position() == Approx(0.0).margin(0.0001)); + CHECK(effect->Order() == 3); + break; + case 2: + CHECK(effect->Layer() == 1); + CHECK(effect->Id() == "B-2"); + CHECK(effect->Position() == Approx(0.5).margin(0.0001)); + CHECK(effect->Order() == 2); + break; + case 3: + CHECK(effect->Layer() == 1); + CHECK(effect->Id() == "B-3"); + CHECK(effect->Position() == Approx(0.5).margin(0.0001)); + CHECK(effect->Order() == 1); + break; + case 4: + CHECK(effect->Layer() == 2); + CHECK(effect->Id() == "C"); + CHECK(effect->Order() == 0); + break; + } + ++n; + } +} - // Calculate test video path +TEST_CASE( "Multi-threaded Timeline Add/Remove Effect", "[libopenshot][timeline]" ) +{ + // Create timeline with a clip so frames can be requested + Timeline *t = new Timeline(1280, 720, Fraction(24, 1), 48000, 2, LAYOUT_STEREO); std::stringstream path; path << TEST_MEDIA_PATH << "test.mp4"; + Clip *clip = new Clip(path.str()); + clip->Layer(0); + t->AddClip(clip); + t->Open(); - // A successful test will NOT crash - since this causes many threads to - // call the same Timeline methods asynchronously, to verify mutexes and multi-threaded - // access does not seg fault or crash this test. + // A successful test will NOT crash - many threads will add/remove effects + // while also requesting frames, exercising locks around effect mutation. #pragma omp parallel { - // Run the following loop in all threads - int64_t clip_count = 10; - for (int clip_index = 1; clip_index <= clip_count; clip_index++) { - // Create clip - Clip* clip_video = new Clip(path.str()); - clip_video->Layer(omp_get_thread_num()); - - // Add clip to timeline - t->AddClip(clip_video); - - // Loop through all timeline frames - each new clip makes the timeline longer - for (long int frame = 10; frame >= 1; frame--) { + int64_t effect_count = 10; + for (int i = 0; i < effect_count; ++i) { + // Each thread creates its own effect + Negate *neg = new Negate(); + std::stringstream sid; + sid << "NEG_T" << omp_get_thread_num() << "_I" << i; + neg->Id(sid.str()); + neg->Layer(1 + omp_get_thread_num()); // spread across layers + + // Add the effect + t->AddEffect(neg); + + // Touch a few frames to exercise the render pipeline with the effect + for (long int frame = 1; frame <= 6; ++frame) { std::shared_ptr f = t->GetFrame(frame); - t->GetMaxFrame(); + REQUIRE(f != nullptr); } - // Remove clip - t->RemoveClip(clip_video); - delete clip_video; - clip_video = NULL; + // Remove the effect and destroy it + t->RemoveEffect(neg); + delete neg; + neg = nullptr; } - // Clear all clips after loop is done - // This is designed to test the mutex for Clear() - t->Clear(); + // Clear all effects at the end from within threads (should be safe) + // This also exercises internal sorting/locking paths + t->Clear(); } - // Close and delete timeline object t->Close(); delete t; - t = NULL; + t = nullptr; + delete clip; + clip = nullptr; } TEST_CASE( "ApplyJSONDiff and FrameMappers", "[libopenshot][timeline]" ) @@ -889,7 +1040,7 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) CHECK(clip1.info.fps.den == 1); CHECK(clip1.info.video_timebase.num == 1); CHECK(clip1.info.video_timebase.den == 24); - CHECK(clip1.info.duration == Approx(51.94667).margin(0.00001)); + CHECK(clip1.info.duration == Approx(52.20833).margin(0.00001)); // Create JSON change to increase FPS from 24 to 60 Json::Value reader_root = openshot::stringToJson(reader_json); @@ -914,14 +1065,14 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) CHECK(mapper->Reader()->info.fps.den == 1); CHECK(mapper->Reader()->info.video_timebase.num == 1); CHECK(mapper->Reader()->info.video_timebase.den == 60); - CHECK(mapper->Reader()->info.duration == Approx(20.77867).margin(0.00001)); + CHECK(mapper->Reader()->info.duration == Approx(20.88333).margin(0.00001)); // Verify clip has updated properties and info struct CHECK(clip1.info.fps.num == 24); CHECK(clip1.info.fps.den == 1); CHECK(clip1.info.video_timebase.num == 1); CHECK(clip1.info.video_timebase.den == 24); - CHECK(clip1.info.duration == Approx(20.77867).margin(0.00001)); + CHECK(clip1.info.duration == Approx(20.88333).margin(0.00001)); // Open Clip object, and verify this does not clobber our 60 FPS change clip1.Open(); @@ -929,6 +1080,6 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) CHECK(mapper->Reader()->info.fps.den == 1); CHECK(mapper->Reader()->info.video_timebase.num == 1); CHECK(mapper->Reader()->info.video_timebase.den == 60); - CHECK(mapper->Reader()->info.duration == Approx(20.77867).margin(0.00001)); + CHECK(mapper->Reader()->info.duration == Approx(20.88333).margin(0.00001)); } diff --git a/tests/WaveEffect.cpp b/tests/WaveEffect.cpp new file mode 100644 index 000000000..da4386c3b --- /dev/null +++ b/tests/WaveEffect.cpp @@ -0,0 +1,41 @@ +/** + * @file + * @brief Unit tests for openshot::Wave effect + * @author OpenAI ChatGPT + */ + +// Copyright (c) 2008-2025 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include + +#include "Frame.h" +#include "effects/Wave.h" +#include "openshot_catch.h" + +using namespace openshot; + +TEST_CASE("Wave uses original pixel buffer", "[effect][wave]") +{ + // Create 1x10 image with increasing red channel + QImage img(10, 1, QImage::Format_ARGB32); + for (int x = 0; x < 10; ++x) + img.setPixelColor(x, 0, QColor(x, 0, 0, 255)); + auto f = std::make_shared(); + *f->GetImage() = img; + + Wave w; + w.wavelength = Keyframe(0.0); + w.amplitude = Keyframe(1.0); + w.multiplier = Keyframe(0.01); + w.shift_x = Keyframe(-1.0); // negative shift to copy from previous pixel + w.speed_y = Keyframe(0.0); + + auto out_img = w.GetFrame(f, 1)->GetImage(); + int expected[10] = {0,0,1,2,3,4,5,6,7,8}; + for (int x = 0; x < 10; ++x) + CHECK(out_img->pixelColor(x,0).red() == expected[x]); +}