From 1414e070328c3ca422f4a042a7deb5c90e272cac Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Mon, 17 Jun 2024 22:18:26 +0200 Subject: [PATCH 01/33] V1 - do nothing smart - all in/all out --- .../media/server/impl/rtp/JitterBuffer.java | 207 ++++++++---------- .../media/server/impl/rtp/RtpHandler.java | 1 - .../server/impl/rtp/JitterBufferTest.java | 1 - 3 files changed, 92 insertions(+), 117 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 25fd3af6e..e270de302 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -32,7 +32,6 @@ import org.apache.logging.log4j.Logger; import org.mobicents.media.server.io.sdp.format.RTPFormat; -import org.mobicents.media.server.io.sdp.format.RTPFormats; import org.mobicents.media.server.scheduler.PriorityQueueScheduler; import org.mobicents.media.server.spi.memory.Frame; @@ -64,7 +63,7 @@ public class JitterBuffer implements Serializable { private final double JC_GAMMA = .01d; //The underlying buffer size - private static final int QUEUE_SIZE = 10; + private static final int QUEUE_SIZE = 20; //the underlying buffer private ArrayList queue = new ArrayList(QUEUE_SIZE); @@ -73,9 +72,6 @@ public class JitterBuffer implements Serializable { //first received sequence number private long isn = -1; - //allowed jitter - private long jitterBufferSize; - //packet arrival dead line measured on RTP clock. //initial value equals to infinity private long arrivalDeadLine = 0; @@ -106,10 +102,7 @@ public class JitterBuffer implements Serializable { * continuously updated value of network jitter */ private long currentJitter = 0; - - //transmission formats - private RTPFormats rtpFormats = new RTPFormats(); - + //currently used format private RTPFormat format; @@ -138,7 +131,6 @@ public class JitterBuffer implements Serializable { */ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler scheduler, Path dumpDir) { this.rtpClock = clock; - this.jitterBufferSize = jitterBufferSize; this.scheduler = scheduler; if (dumpDir != null) { this.dumpDir = dumpDir; @@ -149,44 +141,44 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler } } - private void initJitter(RtpPacket firstPacket) { - long arrival = rtpClock.getLocalRtpTime(); - long firstPacketTimestamp = firstPacket.getTimestamp(); - currentTransit = arrival - firstPacketTimestamp; - currentJitter = 0; - clockOffset = currentTransit; - } - - /** - * Calculates the current network jitter, which is an estimate of the - * statistical variance of the RTP data packet interarrival time: - * http://tools.ietf.org/html/rfc3550#appendix-A.8 - */ - private void estimateJitter(RtpPacket newPacket) { - long arrival = rtpClock.getLocalRtpTime(); - long newPacketTimestamp = newPacket.getTimestamp(); - long transit = arrival - newPacketTimestamp; - long d = transit - currentTransit; - if (d < 0) { - d = -d; - } - - currentTransit = transit; - currentJitter += d - ((currentJitter + 8) >> 4); - - long diff = newPacketTimestamp - arrival; - double slide = (double)clockOffset*(1-JC_BETA) + (diff*JC_BETA); - double gap = diff - slide; - - gap = gap < 0 ? -gap : 0; - jitter = jitter*(1-JC_GAMMA) + (gap*JC_GAMMA); - - if (newPacket.getSeqNumber()%50 == 0) { - adaptJittCompTimestamp = Math.max((int)jittCompTimestamp, (int)(2*jitter)); - } - - clockOffset = (long)slide; - } +// private void initJitter(RtpPacket firstPacket) { +// long arrival = rtpClock.getLocalRtpTime(); +// long firstPacketTimestamp = firstPacket.getTimestamp(); +// currentTransit = arrival - firstPacketTimestamp; +// currentJitter = 0; +// clockOffset = currentTransit; +// } +// +// /** +// * Calculates the current network jitter, which is an estimate of the +// * statistical variance of the RTP data packet interarrival time: +// * http://tools.ietf.org/html/rfc3550#appendix-A.8 +// */ +// private void estimateJitter(RtpPacket newPacket) { +// long arrival = rtpClock.getLocalRtpTime(); +// long newPacketTimestamp = newPacket.getTimestamp(); +// long transit = arrival - newPacketTimestamp; +// long d = transit - currentTransit; +// if (d < 0) { +// d = -d; +// } +// +// currentTransit = transit; +// currentJitter += d - ((currentJitter + 8) >> 4); +// +// long diff = newPacketTimestamp - arrival; +// double slide = (double)clockOffset*(1-JC_BETA) + (diff*JC_BETA); +// double gap = diff - slide; +// +// gap = gap < 0 ? -gap : 0; +// jitter = jitter*(1-JC_GAMMA) + (gap*JC_GAMMA); +// +// if (newPacket.getSeqNumber()%50 == 0) { +// adaptJittCompTimestamp = Math.max((int)jittCompTimestamp, (int)(2*jitter)); +// } +// +// clockOffset = (long)slide; +// } /** * @@ -198,11 +190,7 @@ public long getEstimatedJitter() { // logger.info(String.format("Jitter estimated at %d. Current transit time is %d.", jitterEstimate, currentTransit)); return jitterEstimate; } - - public void setFormats(RTPFormats rtpFormats) { - this.rtpFormats = rtpFormats; - } - + /** * Gets the interarrival jitter. * @@ -212,15 +200,6 @@ public double getJitter() { return 0; } - /** - * Gets the maximum interarrival jitter. - * - * @return the jitter value. - */ - public double getMaxJitter() { - return 0; - } - /** * Get the number of dropped packets. * @@ -230,11 +209,6 @@ public int getDropped() { return dropCount; } - public boolean bufferInUse() - { - return this.useBuffer; - } - public void setBufferInUse(boolean useBuffer) { this.useBuffer=useBuffer; @@ -249,10 +223,10 @@ public void setListener(BufferListener listener) { this.listener = listener; } - private long compensatedTimestamp(long userTimestamp) { - return userTimestamp+clockOffset-adaptJittCompTimestamp; - } - +// private long compensatedTimestamp(long userTimestamp) { +// return userTimestamp+clockOffset-adaptJittCompTimestamp; +// } +// /** * Accepts specified packet * @@ -291,10 +265,11 @@ public void write(RtpPacket packet, RTPFormat format) { if (isn == -1) { rtpClock.synchronize(packet.getTimestamp()); isn = packet.getSeqNumber(); - initJitter(packet); - } else { - estimateJitter(packet); +// initJitter(packet); } +// else { +// estimateJitter(packet); +// } // drop outstanding packets // packet is outstanding if its timestamp of arrived packet is less @@ -308,19 +283,20 @@ public void write(RtpPacket packet, RTPFormat format) { ", format=" + this.format.toString() + ", csrc: " + packet.getContributingSource() ); - dropCount++; - - // checking if not dropping too much - droppedInRaw++; - if (droppedInRaw == QUEUE_SIZE / 2 || queue.size() == 0) { - arrivalDeadLine = 0; - } else { - return; - } +// dropCount++; +// +// // checking if not dropping too much +// droppedInRaw++; +// if (droppedInRaw == QUEUE_SIZE / 2 || queue.size() == 0) { +// arrivalDeadLine = 0; +// } else { +// return; +// } + return; } Frame f = packet.toFrame(rtpClock, this.format); -f.setDuration(rtpClock.convertToAbsoluteTime(f.getLength())); + f.setDuration(rtpClock.convertToAbsoluteTime(f.getLength())); // dump the packet to capture if enabled so if (this.dumpConfig != null) { @@ -373,20 +349,20 @@ public void write(RtpPacket packet, RTPFormat format) { return; } - // overflow? - // only now remove packet if overflow , possibly the same packet we just received - if (queue.size() > QUEUE_SIZE) { - logger.warn("Buffer overflow!" + - " queue: " + queue.size() + - ", localPeer: " + (packet.getLocalPeer() != null ? packet.getLocalPeer().toString() : "null") + - ", remotePeer: " + (packet.getRemotePeer() != null ? packet.getRemotePeer().toString() : "null") + - ", seq: " + packet.getSeqNumber() + - ", timestamp: " + packet.getTimestamp() + - ", csrc: " + packet.getContributingSource() - ); - dropCount++; - queue.remove(0); - } +// // overflow? +// // only now remove packet if overflow , possibly the same packet we just received +// if (queue.size() > QUEUE_SIZE) { +// logger.warn("Buffer overflow!" + +// " queue: " + queue.size() + +// ", localPeer: " + (packet.getLocalPeer() != null ? packet.getLocalPeer().toString() : "null") + +// ", remotePeer: " + (packet.getRemotePeer() != null ? packet.getRemotePeer().toString() : "null") + +// ", seq: " + packet.getSeqNumber() + +// ", timestamp: " + packet.getTimestamp() + +// ", csrc: " + packet.getContributingSource() +// ); +// dropCount++; +// queue.remove(0); +// } // check if this buffer already full if (!ready) { @@ -415,19 +391,19 @@ public Frame read(long timestamp) { return null; } - Frame frame = null; - long rtpTime; - - long comp = compensatedTimestamp(rtpClock.getLocalRtpTime()); - - while (queue.size() != 0) { - frame = queue.remove(0); - rtpTime = rtpClock.convertToRtpTime(frame.getTimestamp()); - - if (comp <= rtpTime) { - break; - } - } + Frame frame = queue.remove(0); +// long rtpTime; + +// long comp = compensatedTimestamp(rtpClock.getLocalRtpTime()); +// +// while (queue.size() != 0) { +// frame = queue.remove(0); +// rtpTime = rtpClock.convertToRtpTime(frame.getTimestamp()); +// +// if (comp <= rtpTime) { +// break; +// } +// } if (this.dumpConfig != null) { JitterBufferRTPDump dump = rtpDump.get(); @@ -437,11 +413,12 @@ public Frame read(long timestamp) { } } - if (frame == null) { - return null; - } +// if (frame == null) { +// this.ready = false; +// return null; +// } - //buffer empty now? - change ready flag. +// //buffer empty now? - change ready flag. if (queue.size() == 0) { this.ready = false; } diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java index f3be05b5d..553cfd049 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java @@ -132,7 +132,6 @@ public void useJitterBuffer(boolean useBuffer) { */ public void setFormatMap(final RTPFormats rtpFormats) { this.rtpFormats = rtpFormats; - this.jitterBuffer.setFormats(rtpFormats); } public RTPFormats getFormatMap() { diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index cfe4a49c0..9e07244e5 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -64,7 +64,6 @@ public static void tearDownClass() throws Exception { @Before public void setUp() { - jitterBuffer.setFormats(AVProfile.audio); rtpClock.setClockRate(8000); jitterBuffer.reset(); } From 5d139a88364ddd59e66ab9d92294df264438539f Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 18 Jun 2024 11:15:17 +0200 Subject: [PATCH 02/33] v2 - make it work? --- .../media/server/impl/rtp/JitterBuffer.java | 68 ++++++++++++++++++- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index e270de302..2dd73e512 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -25,6 +25,7 @@ import java.io.Serializable; import java.nio.file.Path; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -62,6 +63,18 @@ public class JitterBuffer implements Serializable { private final double JC_BETA = .01d; private final double JC_GAMMA = .01d; + private final int BUFFER_SIZE_MAX = 10; + private final int BUFFER_SIZE_NOR = 5; + private final int BUFFER_SIZE_MIN = 1; + + private final double SPEED_FAST = 0.3; + private final double SPEED_NOR = 0.8; + private final double SPEED_SLOW = 1.5; + private final int NUM_FRAME_TIME_HISTORY = 60; + private double avgFrameRate; + private double lastFrameRate; + private LinkedList decodedFrameTime = new LinkedList<>(); + //The underlying buffer size private static final int QUEUE_SIZE = 20; //the underlying buffer @@ -132,6 +145,10 @@ public class JitterBuffer implements Serializable { public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler scheduler, Path dumpDir) { this.rtpClock = clock; this.scheduler = scheduler; + this.lastFrameRate = 1.0f; + this.avgFrameRate = 1.0f; + + this.decodedFrameTime.push(0L); if (dumpDir != null) { this.dumpDir = dumpDir; this.dumpConfig = JitterBufferRTPDump.getDumpConfig(dumpDir); @@ -386,11 +403,45 @@ public void write(RtpPacket packet, RTPFormat format) { public Frame read(long timestamp) { try { LOCK.lock(); - if (queue.size() == 0) { + + int size = queue.size(); + + long currentTime = rtpClock.getLocalRtpTime(); + + if (size < BUFFER_SIZE_MIN) { this.ready = false; return null; } + if (size < BUFFER_SIZE_NOR) { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / avgFrameRate)) + { + this.ready = false; + return null; + } + + } + + if (size < BUFFER_SIZE_MAX) + { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / avgFrameRate) && + (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) + { + this.ready = false; + return null; + } + } + + if (size >= BUFFER_SIZE_MAX) + { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST/ lastFrameRate) && + (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / avgFrameRate)) + { + this.ready = false; + return null; + } + } + Frame frame = queue.remove(0); // long rtpTime; @@ -419,7 +470,7 @@ public Frame read(long timestamp) { // } // //buffer empty now? - change ready flag. - if (queue.size() == 0) { + if (size == 1) { this.ready = false; } @@ -429,6 +480,15 @@ public Frame read(long timestamp) { frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); + lastFrameRate = 1000.0 / (currentTime - decodedFrameTime.peekFirst()); + decodedFrameTime.push(currentTime); + avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); + + if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) + { + decodedFrameTime.removeLast(); + } + return frame; } finally { LOCK.unlock(); @@ -474,6 +534,10 @@ public void restart() { adaptJittCompTimestamp = 0; jittCompTimestamp = 0; jitter = 0d; + lastFrameRate = 1.0f; + avgFrameRate = 1.0f; + + decodedFrameTime.push(0L); restartRecording(); } From 14172c3728cdde3ee34d93ccde076e7af9d1ae0c Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Tue, 18 Jun 2024 11:40:52 +0200 Subject: [PATCH 03/33] Fix starting decodeFrameTime. --- .../media/server/impl/rtp/JitterBuffer.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 2dd73e512..a20a7d91c 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -148,7 +148,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.lastFrameRate = 1.0f; this.avgFrameRate = 1.0f; - this.decodedFrameTime.push(0L); + this.decodedFrameTime.push(System.currentTimeMillis()); if (dumpDir != null) { this.dumpDir = dumpDir; this.dumpConfig = JitterBufferRTPDump.getDumpConfig(dumpDir); @@ -259,6 +259,8 @@ public void write(RtpPacket packet, RTPFormat format) { } +// System.out.println("XXXX WRITING PACKET: "); + if (this.format == null || this.format.getID() != format.getID()) { logger.info( @@ -406,16 +408,26 @@ public Frame read(long timestamp) { int size = queue.size(); - long currentTime = rtpClock.getLocalRtpTime(); + long currentTime = System.currentTimeMillis(); + +// System.out.println("XXXX READING PACKET: " + timestamp); + + if (size < BUFFER_SIZE_MIN) { this.ready = false; +// System.out.println("XXXX NULL 1 "); return null; } + +// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + if (size < BUFFER_SIZE_NOR) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / avgFrameRate)) { +// System.out.println("XXXX NULL 2 "); + this.ready = false; return null; } @@ -427,6 +439,8 @@ public Frame read(long timestamp) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / avgFrameRate) && (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) { +// System.out.println("XXXX NULL 3 "); + this.ready = false; return null; } @@ -437,6 +451,8 @@ public Frame read(long timestamp) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST/ lastFrameRate) && (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / avgFrameRate)) { +// System.out.println("XXXX NULL 4 "); + this.ready = false; return null; } @@ -474,6 +490,10 @@ public Frame read(long timestamp) { this.ready = false; } +// System.out.println("XXXX READING AND WILL RETURN: "); + + + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); //convert duration to nanoseconds @@ -484,6 +504,9 @@ public Frame read(long timestamp) { decodedFrameTime.push(currentTime); avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); +// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); + + if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) { decodedFrameTime.removeLast(); @@ -537,7 +560,7 @@ public void restart() { lastFrameRate = 1.0f; avgFrameRate = 1.0f; - decodedFrameTime.push(0L); + decodedFrameTime.push(System.currentTimeMillis()); restartRecording(); } From 1b4ed8fe0b2e2cd571c303e49f2a3effacb7bec6 Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Tue, 18 Jun 2024 11:46:28 +0200 Subject: [PATCH 04/33] Tweak params. --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index a20a7d91c..b3be0eb58 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -63,8 +63,8 @@ public class JitterBuffer implements Serializable { private final double JC_BETA = .01d; private final double JC_GAMMA = .01d; - private final int BUFFER_SIZE_MAX = 10; - private final int BUFFER_SIZE_NOR = 5; + private final int BUFFER_SIZE_MAX = 6; + private final int BUFFER_SIZE_NOR = 3; private final int BUFFER_SIZE_MIN = 1; private final double SPEED_FAST = 0.3; From 8fc09796b2b20ac98382bbbcef33e4a54db716f5 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 18 Jun 2024 13:23:22 +0200 Subject: [PATCH 05/33] cleanup & minFrameRate --- .../media/server/impl/rtp/JitterBuffer.java | 142 +----------------- 1 file changed, 8 insertions(+), 134 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index b3be0eb58..0feb14069 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -60,9 +60,6 @@ public class JitterBuffer implements Serializable { private final ReentrantLock LOCK = new ReentrantLock(); - private final double JC_BETA = .01d; - private final double JC_GAMMA = .01d; - private final int BUFFER_SIZE_MAX = 6; private final int BUFFER_SIZE_NOR = 3; private final int BUFFER_SIZE_MIN = 1; @@ -73,6 +70,7 @@ public class JitterBuffer implements Serializable { private final int NUM_FRAME_TIME_HISTORY = 60; private double avgFrameRate; private double lastFrameRate; + private double minFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); //The underlying buffer size @@ -89,9 +87,6 @@ public class JitterBuffer implements Serializable { //initial value equals to infinity private long arrivalDeadLine = 0; - //packet arrival dead line measured on RTP clock. - //initial value equals to infinity - private long droppedInRaw = 0; //The number of dropped packets private int dropCount; @@ -104,13 +99,6 @@ public class JitterBuffer implements Serializable { private volatile boolean ready; - /** - * used to calculate network jitter. - * currentTransit measures the relative time it takes for an RTP packet - * to arrive from the remote server to MMS - */ - private long currentTransit = 0; - /** * continuously updated value of network jitter */ @@ -123,11 +111,6 @@ public class JitterBuffer implements Serializable { private final static Logger logger = org.apache.logging.log4j.LogManager.getLogger(JitterBuffer.class); - private long clockOffset = 0; - private int adaptJittCompTimestamp = 0; - private long jittCompTimestamp = 0; - private double jitter = 0d; - private PriorityQueueScheduler scheduler; private static AtomicLong recordingIndex = new AtomicLong(); @@ -147,6 +130,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.scheduler = scheduler; this.lastFrameRate = 1.0f; this.avgFrameRate = 1.0f; + this.minFrameRate = 50; this.decodedFrameTime.push(System.currentTimeMillis()); if (dumpDir != null) { @@ -158,45 +142,6 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler } } -// private void initJitter(RtpPacket firstPacket) { -// long arrival = rtpClock.getLocalRtpTime(); -// long firstPacketTimestamp = firstPacket.getTimestamp(); -// currentTransit = arrival - firstPacketTimestamp; -// currentJitter = 0; -// clockOffset = currentTransit; -// } -// -// /** -// * Calculates the current network jitter, which is an estimate of the -// * statistical variance of the RTP data packet interarrival time: -// * http://tools.ietf.org/html/rfc3550#appendix-A.8 -// */ -// private void estimateJitter(RtpPacket newPacket) { -// long arrival = rtpClock.getLocalRtpTime(); -// long newPacketTimestamp = newPacket.getTimestamp(); -// long transit = arrival - newPacketTimestamp; -// long d = transit - currentTransit; -// if (d < 0) { -// d = -d; -// } -// -// currentTransit = transit; -// currentJitter += d - ((currentJitter + 8) >> 4); -// -// long diff = newPacketTimestamp - arrival; -// double slide = (double)clockOffset*(1-JC_BETA) + (diff*JC_BETA); -// double gap = diff - slide; -// -// gap = gap < 0 ? -gap : 0; -// jitter = jitter*(1-JC_GAMMA) + (gap*JC_GAMMA); -// -// if (newPacket.getSeqNumber()%50 == 0) { -// adaptJittCompTimestamp = Math.max((int)jittCompTimestamp, (int)(2*jitter)); -// } -// -// clockOffset = (long)slide; -// } - /** * * @return the current value of the network RTP jitter. The value is in normalized form as specified in RFC 3550 @@ -208,15 +153,6 @@ public long getEstimatedJitter() { return jitterEstimate; } - /** - * Gets the interarrival jitter. - * - * @return the current jitter value. - */ - public double getJitter() { - return 0; - } - /** * Get the number of dropped packets. * @@ -240,10 +176,6 @@ public void setListener(BufferListener listener) { this.listener = listener; } -// private long compensatedTimestamp(long userTimestamp) { -// return userTimestamp+clockOffset-adaptJittCompTimestamp; -// } -// /** * Accepts specified packet * @@ -277,18 +209,13 @@ public void write(RtpPacket packet, RTPFormat format) { // update clock rate rtpClock.setClockRate(this.format.getClockRate()); - jittCompTimestamp = rtpClock.convertToRtpTime(60); } // if this is first packet then synchronize clock if (isn == -1) { rtpClock.synchronize(packet.getTimestamp()); isn = packet.getSeqNumber(); -// initJitter(packet); } -// else { -// estimateJitter(packet); -// } // drop outstanding packets // packet is outstanding if its timestamp of arrived packet is less @@ -302,15 +229,7 @@ public void write(RtpPacket packet, RTPFormat format) { ", format=" + this.format.toString() + ", csrc: " + packet.getContributingSource() ); -// dropCount++; -// -// // checking if not dropping too much -// droppedInRaw++; -// if (droppedInRaw == QUEUE_SIZE / 2 || queue.size() == 0) { -// arrivalDeadLine = 0; -// } else { -// return; -// } + return; } @@ -323,8 +242,6 @@ public void write(RtpPacket packet, RTPFormat format) { if (dump != null) dump.dump(packet, queue.size()); } - droppedInRaw = 0; - // find correct position to insert a packet // use timestamp since its always positive int currIndex = queue.size() - 1; @@ -353,13 +270,6 @@ public void write(RtpPacket packet, RTPFormat format) { duration = queue.get(queue.size() - 1).getTimestamp() - queue.get(0).getTimestamp(); } -// for (int i = 0; i < queue.size() - 1; i++) { -// // duration measured by wall clock -// long d = queue.get(i + 1).getTimestamp() - queue.get(i).getTimestamp(); -// // in case of RFC2833 event timestamp remains same -// queue.get(i).setDuration(d > 0 ? d : 0); -// } - // if overall duration is negative we have some mess here,try to // reset if (duration < 0 && queue.size() > 1) { @@ -368,21 +278,6 @@ public void write(RtpPacket packet, RTPFormat format) { return; } -// // overflow? -// // only now remove packet if overflow , possibly the same packet we just received -// if (queue.size() > QUEUE_SIZE) { -// logger.warn("Buffer overflow!" + -// " queue: " + queue.size() + -// ", localPeer: " + (packet.getLocalPeer() != null ? packet.getLocalPeer().toString() : "null") + -// ", remotePeer: " + (packet.getRemotePeer() != null ? packet.getRemotePeer().toString() : "null") + -// ", seq: " + packet.getSeqNumber() + -// ", timestamp: " + packet.getTimestamp() + -// ", csrc: " + packet.getContributingSource() -// ); -// dropCount++; -// queue.remove(0); -// } - // check if this buffer already full if (!ready) { ready = !useBuffer || (queue.size() > 1); @@ -424,7 +319,7 @@ public Frame read(long timestamp) { // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); if (size < BUFFER_SIZE_NOR) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / avgFrameRate)) + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / minFrameRate)) { // System.out.println("XXXX NULL 2 "); @@ -436,7 +331,7 @@ public Frame read(long timestamp) { if (size < BUFFER_SIZE_MAX) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / avgFrameRate) && + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / minFrameRate) && (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) { // System.out.println("XXXX NULL 3 "); @@ -449,7 +344,7 @@ public Frame read(long timestamp) { if (size >= BUFFER_SIZE_MAX) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST/ lastFrameRate) && - (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / avgFrameRate)) + (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / minFrameRate)) { // System.out.println("XXXX NULL 4 "); @@ -459,18 +354,6 @@ public Frame read(long timestamp) { } Frame frame = queue.remove(0); -// long rtpTime; - -// long comp = compensatedTimestamp(rtpClock.getLocalRtpTime()); -// -// while (queue.size() != 0) { -// frame = queue.remove(0); -// rtpTime = rtpClock.convertToRtpTime(frame.getTimestamp()); -// -// if (comp <= rtpTime) { -// break; -// } -// } if (this.dumpConfig != null) { JitterBufferRTPDump dump = rtpDump.get(); @@ -480,11 +363,6 @@ public Frame read(long timestamp) { } } -// if (frame == null) { -// this.ready = false; -// return null; -// } - // //buffer empty now? - change ready flag. if (size == 1) { this.ready = false; @@ -503,6 +381,7 @@ public Frame read(long timestamp) { lastFrameRate = 1000.0 / (currentTime - decodedFrameTime.peekFirst()); decodedFrameTime.push(currentTime); avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); + minFrameRate = Math.min(minFrameRate, lastFrameRate); // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -548,17 +427,12 @@ public void restart() { this.ready=false; arrivalDeadLine = 0; dropCount=0; - droppedInRaw=0; format=null; isn=-1; - // - clockOffset = 0; - adaptJittCompTimestamp = 0; - jittCompTimestamp = 0; - jitter = 0d; lastFrameRate = 1.0f; avgFrameRate = 1.0f; + minFrameRate = 50; decodedFrameTime.push(System.currentTimeMillis()); From 161032238c347bdb9ec9acc6fbc9b86ef4259acd Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 18 Jun 2024 13:52:06 +0200 Subject: [PATCH 06/33] useBuffer --- .../media/server/impl/rtp/JitterBuffer.java | 117 ++++++++++-------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 0feb14069..24ed26de1 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -300,98 +300,107 @@ public void write(RtpPacket packet, RTPFormat format) { public Frame read(long timestamp) { try { LOCK.lock(); + if (!useBuffer) { + if (queue.isEmpty()) { + return null; + } else { + Frame frame = queue.remove(0); - int size = queue.size(); + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); - long currentTime = System.currentTimeMillis(); + //convert duration to nanoseconds + frame.setDuration(frame.getDuration() * 1000000L); + frame.setTimestamp(frame.getTimestamp() * 1000000L); -// System.out.println("XXXX READING PACKET: " + timestamp); + return frame; + } + + } else { + int size = queue.size(); + long currentTime = System.currentTimeMillis(); - if (size < BUFFER_SIZE_MIN) { - this.ready = false; +// System.out.println("XXXX READING PACKET: " + timestamp); + + + if (size < BUFFER_SIZE_MIN) { + this.ready = false; // System.out.println("XXXX NULL 1 "); - return null; - } + return null; + } // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - if (size < BUFFER_SIZE_NOR) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / minFrameRate)) - { + if (size < BUFFER_SIZE_NOR) { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / minFrameRate)) { // System.out.println("XXXX NULL 2 "); - this.ready = false; - return null; - } + this.ready = false; + return null; + } - } + } - if (size < BUFFER_SIZE_MAX) - { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / minFrameRate) && - (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) - { + if (size < BUFFER_SIZE_MAX) { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / minFrameRate) && + (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) { // System.out.println("XXXX NULL 3 "); - this.ready = false; - return null; + this.ready = false; + return null; + } } - } - if (size >= BUFFER_SIZE_MAX) - { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST/ lastFrameRate) && - (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / minFrameRate)) - { + if (size >= BUFFER_SIZE_MAX) { + if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / lastFrameRate) && + (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / minFrameRate)) { // System.out.println("XXXX NULL 4 "); - this.ready = false; - return null; + this.ready = false; + return null; + } } - } - Frame frame = queue.remove(0); + Frame frame = queue.remove(0); - if (this.dumpConfig != null) { - JitterBufferRTPDump dump = rtpDump.get(); - if (dump != null) { - long seq = frame != null ? frame.getSequenceNumber() : -1; - dump.suppliedDump(seq, queue.size()); + if (this.dumpConfig != null) { + JitterBufferRTPDump dump = rtpDump.get(); + if (dump != null) { + long seq = frame != null ? frame.getSequenceNumber() : -1; + dump.suppliedDump(seq, queue.size()); + } } - } // //buffer empty now? - change ready flag. - if (size == 1) { - this.ready = false; - } + if (size == 1) { + this.ready = false; + } // System.out.println("XXXX READING AND WILL RETURN: "); + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + //convert duration to nanoseconds + frame.setDuration(frame.getDuration() * 1000000L); + frame.setTimestamp(frame.getTimestamp() * 1000000L); - //convert duration to nanoseconds - frame.setDuration(frame.getDuration() * 1000000L); - frame.setTimestamp(frame.getTimestamp() * 1000000L); - - lastFrameRate = 1000.0 / (currentTime - decodedFrameTime.peekFirst()); - decodedFrameTime.push(currentTime); - avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); - minFrameRate = Math.min(minFrameRate, lastFrameRate); + lastFrameRate = 1000.0 / (currentTime - decodedFrameTime.peekFirst()); + decodedFrameTime.push(currentTime); + avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); + minFrameRate = Math.min(minFrameRate, lastFrameRate); // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); - if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) - { - decodedFrameTime.removeLast(); - } + if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) { + decodedFrameTime.removeLast(); + } - return frame; + return frame; + } } finally { LOCK.unlock(); } From f16e5e890354ac709f5984317fcf598b4f3f4c7d Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 18 Jun 2024 13:53:25 +0200 Subject: [PATCH 07/33] remove ready --- .../media/server/impl/rtp/JitterBuffer.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 24ed26de1..e8334c513 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -97,8 +97,6 @@ public class JitterBuffer implements Serializable { //buffer's monitor private BufferListener listener; - private volatile boolean ready; - /** * continuously updated value of network jitter */ @@ -278,14 +276,6 @@ public void write(RtpPacket packet, RTPFormat format) { return; } - // check if this buffer already full - if (!ready) { - ready = !useBuffer || (queue.size() > 1); - if (ready && listener != null) { - listener.onFill(); - } - } - } finally { LOCK.unlock(); } @@ -325,7 +315,6 @@ public Frame read(long timestamp) { if (size < BUFFER_SIZE_MIN) { - this.ready = false; // System.out.println("XXXX NULL 1 "); return null; } @@ -336,8 +325,6 @@ public Frame read(long timestamp) { if (size < BUFFER_SIZE_NOR) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / minFrameRate)) { // System.out.println("XXXX NULL 2 "); - - this.ready = false; return null; } @@ -347,8 +334,6 @@ public Frame read(long timestamp) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / minFrameRate) && (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) { // System.out.println("XXXX NULL 3 "); - - this.ready = false; return null; } } @@ -357,8 +342,6 @@ public Frame read(long timestamp) { if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / lastFrameRate) && (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / minFrameRate)) { // System.out.println("XXXX NULL 4 "); - - this.ready = false; return null; } } @@ -373,11 +356,6 @@ public Frame read(long timestamp) { } } -// //buffer empty now? - change ready flag. - if (size == 1) { - this.ready = false; - } - // System.out.println("XXXX READING AND WILL RETURN: "); @@ -433,7 +411,6 @@ private void restartRecording() { public void restart() { reset(); - this.ready=false; arrivalDeadLine = 0; dropCount=0; format=null; From 3eaffe937e0173dfe4503be2ab0eeac47fd89629 Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Tue, 18 Jun 2024 13:57:14 +0200 Subject: [PATCH 08/33] Update constants for speed values. --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index e8334c513..973e13dcb 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -64,9 +64,9 @@ public class JitterBuffer implements Serializable { private final int BUFFER_SIZE_NOR = 3; private final int BUFFER_SIZE_MIN = 1; - private final double SPEED_FAST = 0.3; - private final double SPEED_NOR = 0.8; - private final double SPEED_SLOW = 1.5; + private final double SPEED_FAST = 0.02; + private final double SPEED_NOR = 0.05; + private final double SPEED_SLOW = 0.3; private final int NUM_FRAME_TIME_HISTORY = 60; private double avgFrameRate; private double lastFrameRate; @@ -313,6 +313,8 @@ public Frame read(long timestamp) { // System.out.println("XXXX READING PACKET: " + timestamp); +// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + if (size < BUFFER_SIZE_MIN) { // System.out.println("XXXX NULL 1 "); From fe42e01dd43ac7f3a2d0df4877195144972ee614 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 11:02:47 +0200 Subject: [PATCH 09/33] avg is back --- .../media/server/impl/rtp/JitterBuffer.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 973e13dcb..77f1c12e0 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -70,7 +70,6 @@ public class JitterBuffer implements Serializable { private final int NUM_FRAME_TIME_HISTORY = 60; private double avgFrameRate; private double lastFrameRate; - private double minFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); //The underlying buffer size @@ -128,7 +127,6 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.scheduler = scheduler; this.lastFrameRate = 1.0f; this.avgFrameRate = 1.0f; - this.minFrameRate = 50; this.decodedFrameTime.push(System.currentTimeMillis()); if (dumpDir != null) { @@ -309,23 +307,24 @@ public Frame read(long timestamp) { int size = queue.size(); - long currentTime = System.currentTimeMillis(); - -// System.out.println("XXXX READING PACKET: " + timestamp); - -// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - if (size < BUFFER_SIZE_MIN) { // System.out.println("XXXX NULL 1 "); return null; } + long currentTime = System.currentTimeMillis(); + long currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); + +// System.out.println("XXXX READING PACKET: " + timestamp); + +// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); if (size < BUFFER_SIZE_NOR) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_SLOW / minFrameRate)) { + if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { // System.out.println("XXXX NULL 2 "); return null; } @@ -333,16 +332,16 @@ public Frame read(long timestamp) { } if (size < BUFFER_SIZE_MAX) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / minFrameRate) && - (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_NOR / lastFrameRate)) { + if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && + currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { // System.out.println("XXXX NULL 3 "); return null; } } if (size >= BUFFER_SIZE_MAX) { - if ((currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / lastFrameRate) && - (currentTime - decodedFrameTime.peekFirst()) < (1000 * SPEED_FAST / minFrameRate)) { + if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && + currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { // System.out.println("XXXX NULL 4 "); return null; } @@ -367,11 +366,9 @@ public Frame read(long timestamp) { frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); - lastFrameRate = 1000.0 / (currentTime - decodedFrameTime.peekFirst()); + lastFrameRate = 1000.0 / currentTimeDiff; decodedFrameTime.push(currentTime); avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); - minFrameRate = Math.min(minFrameRate, lastFrameRate); - // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -420,7 +417,6 @@ public void restart() { lastFrameRate = 1.0f; avgFrameRate = 1.0f; - minFrameRate = 50; decodedFrameTime.push(System.currentTimeMillis()); From 4f359d1a227a71333dc856c103bf37b770a0e12f Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 11:15:02 +0200 Subject: [PATCH 10/33] simple frame rate comparison --- .../media/server/impl/rtp/JitterBuffer.java | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 77f1c12e0..e5c205b23 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -126,7 +126,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.rtpClock = clock; this.scheduler = scheduler; this.lastFrameRate = 1.0f; - this.avgFrameRate = 1.0f; + this.avgFrameRate = 50.0f; this.decodedFrameTime.push(System.currentTimeMillis()); if (dumpDir != null) { @@ -315,6 +315,7 @@ public Frame read(long timestamp) { long currentTime = System.currentTimeMillis(); long currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); + long currentTimeFrameRate = 1000 / currentTimeDiff; // System.out.println("XXXX READING PACKET: " + timestamp); @@ -323,28 +324,33 @@ public Frame read(long timestamp) { // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - if (size < BUFFER_SIZE_NOR) { - if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { +// if (size < BUFFER_SIZE_NOR) { +// if (currentTimeFrameRate > avgFrameRate) { +//// System.out.println("XXXX NULL 2 "); +// return null; +// } +// +// } +// +// if (size < BUFFER_SIZE_MAX) { +// if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && +// currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { +//// System.out.println("XXXX NULL 3 "); +// return null; +// } +// } +// +// if (size >= BUFFER_SIZE_MAX) { +// if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && +// currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { +//// System.out.println("XXXX NULL 4 "); +// return null; +// } +// } + + if (currentTimeFrameRate > avgFrameRate) { // System.out.println("XXXX NULL 2 "); - return null; - } - - } - - if (size < BUFFER_SIZE_MAX) { - if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && - currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { -// System.out.println("XXXX NULL 3 "); - return null; - } - } - - if (size >= BUFFER_SIZE_MAX) { - if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && - currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { -// System.out.println("XXXX NULL 4 "); - return null; - } + return null; } Frame frame = queue.remove(0); @@ -366,7 +372,7 @@ public Frame read(long timestamp) { frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); - lastFrameRate = 1000.0 / currentTimeDiff; + lastFrameRate = currentTimeFrameRate; decodedFrameTime.push(currentTime); avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -416,7 +422,7 @@ public void restart() { isn=-1; lastFrameRate = 1.0f; - avgFrameRate = 1.0f; + avgFrameRate = 50.0f; decodedFrameTime.push(System.currentTimeMillis()); From ac54f0a7459a3d6b9dd805edc0c50323f4abdca8 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 11:20:07 +0200 Subject: [PATCH 11/33] compare relative to size of the queue --- .../media/server/impl/rtp/JitterBuffer.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index e5c205b23..a71e5a08a 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -64,8 +64,8 @@ public class JitterBuffer implements Serializable { private final int BUFFER_SIZE_NOR = 3; private final int BUFFER_SIZE_MIN = 1; - private final double SPEED_FAST = 0.02; - private final double SPEED_NOR = 0.05; + private final double SPEED_FAST = 2; + private final double SPEED_NOR = 1.5; private final double SPEED_SLOW = 0.3; private final int NUM_FRAME_TIME_HISTORY = 60; private double avgFrameRate; @@ -324,35 +324,39 @@ public Frame read(long timestamp) { // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); -// if (size < BUFFER_SIZE_NOR) { -// if (currentTimeFrameRate > avgFrameRate) { -//// System.out.println("XXXX NULL 2 "); -// return null; -// } -// -// } -// -// if (size < BUFFER_SIZE_MAX) { + if (size < BUFFER_SIZE_NOR) { + if (currentTimeFrameRate > avgFrameRate) { +// System.out.println("XXXX NULL 2 "); + return null; + } + + } + + if (size < BUFFER_SIZE_MAX) { + if (currentTimeFrameRate > (SPEED_NOR*avgFrameRate)) { +// System.out.println("XXXX NULL 2 "); + return null; + } // if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && // currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { //// System.out.println("XXXX NULL 3 "); // return null; // } -// } -// -// if (size >= BUFFER_SIZE_MAX) { + } + + if (size >= BUFFER_SIZE_MAX) { + if (currentTimeFrameRate > SPEED_FAST*avgFrameRate) { +// System.out.println("XXXX NULL 2 "); + return null; + } // if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && // currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { //// System.out.println("XXXX NULL 4 "); // return null; // } -// } - - if (currentTimeFrameRate > avgFrameRate) { -// System.out.println("XXXX NULL 2 "); - return null; } + Frame frame = queue.remove(0); if (this.dumpConfig != null) { From 160e0b6aad2dd5ba54b37d96799eab224bf4f32f Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 13:39:10 +0200 Subject: [PATCH 12/33] use timestamp --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index a71e5a08a..380eb0e91 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -128,7 +128,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.lastFrameRate = 1.0f; this.avgFrameRate = 50.0f; - this.decodedFrameTime.push(System.currentTimeMillis()); + this.decodedFrameTime.push(0L); if (dumpDir != null) { this.dumpDir = dumpDir; this.dumpConfig = JitterBufferRTPDump.getDumpConfig(dumpDir); @@ -313,7 +313,7 @@ public Frame read(long timestamp) { return null; } - long currentTime = System.currentTimeMillis(); + long currentTime = timestamp + 20000; long currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); long currentTimeFrameRate = 1000 / currentTimeDiff; @@ -428,7 +428,7 @@ public void restart() { lastFrameRate = 1.0f; avgFrameRate = 50.0f; - decodedFrameTime.push(System.currentTimeMillis()); + decodedFrameTime.push(0L); restartRecording(); } From 1227544f61ffc5731bc4797e0b70e74235d1aa90 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 13:45:36 +0200 Subject: [PATCH 13/33] avg frame rate is long --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 380eb0e91..35d12749a 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -68,7 +68,7 @@ public class JitterBuffer implements Serializable { private final double SPEED_NOR = 1.5; private final double SPEED_SLOW = 0.3; private final int NUM_FRAME_TIME_HISTORY = 60; - private double avgFrameRate; + private long avgFrameRate; private double lastFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); @@ -126,7 +126,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.rtpClock = clock; this.scheduler = scheduler; this.lastFrameRate = 1.0f; - this.avgFrameRate = 50.0f; + this.avgFrameRate = 50L; this.decodedFrameTime.push(0L); if (dumpDir != null) { @@ -378,7 +378,7 @@ public Frame read(long timestamp) { lastFrameRate = currentTimeFrameRate; decodedFrameTime.push(currentTime); - avgFrameRate = decodedFrameTime.size() * 1000.0 / (currentTime - decodedFrameTime.peekLast()); + avgFrameRate = (currentTime - decodedFrameTime.peekLast()) / decodedFrameTime.size() ; // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -426,7 +426,7 @@ public void restart() { isn=-1; lastFrameRate = 1.0f; - avgFrameRate = 50.0f; + avgFrameRate = 50L; decodedFrameTime.push(0L); From 423ef6b841b8b557163d6ce93ccba22baf287de7 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 13:47:17 +0200 Subject: [PATCH 14/33] make currentTime millis --- .../java/org/mobicents/media/server/impl/rtp/JitterBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 35d12749a..fcc5b616a 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -313,7 +313,7 @@ public Frame read(long timestamp) { return null; } - long currentTime = timestamp + 20000; + long currentTime = timestamp/1000 + 20; long currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); long currentTimeFrameRate = 1000 / currentTimeDiff; From 6df7e2b48bc81e9f21d97dfe1ca50ab10d9aefe6 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 14:04:02 +0200 Subject: [PATCH 15/33] fix avg framerate --- .../media/server/impl/rtp/JitterBuffer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index fcc5b616a..1e334e31e 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -128,7 +128,6 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.lastFrameRate = 1.0f; this.avgFrameRate = 50L; - this.decodedFrameTime.push(0L); if (dumpDir != null) { this.dumpDir = dumpDir; this.dumpConfig = JitterBufferRTPDump.getDumpConfig(dumpDir); @@ -313,8 +312,13 @@ public Frame read(long timestamp) { return null; } - long currentTime = timestamp/1000 + 20; - long currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); + long currentTime = System.currentTimeMillis(); + long currentTimeDiff = 20; + + if (!decodedFrameTime.isEmpty()) { + currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); + } + long currentTimeFrameRate = 1000 / currentTimeDiff; // System.out.println("XXXX READING PACKET: " + timestamp); @@ -378,7 +382,8 @@ public Frame read(long timestamp) { lastFrameRate = currentTimeFrameRate; decodedFrameTime.push(currentTime); - avgFrameRate = (currentTime - decodedFrameTime.peekLast()) / decodedFrameTime.size() ; + avgFrameRate = (currentTime - decodedFrameTime.peekLast()) / decodedFrameTime.size(); + if (avgFrameRate == 0) avgFrameRate = 50; // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -428,7 +433,7 @@ public void restart() { lastFrameRate = 1.0f; avgFrameRate = 50L; - decodedFrameTime.push(0L); + decodedFrameTime.clear(); restartRecording(); } From 965d69ae74cda5eae0c701b3d3a0b4a97c2487dd Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 14:13:38 +0200 Subject: [PATCH 16/33] fix zero in currentTimeDiff --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 1e334e31e..6ece22f3e 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -319,7 +319,11 @@ public Frame read(long timestamp) { currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); } - long currentTimeFrameRate = 1000 / currentTimeDiff; + long currentTimeFrameRate = 50; + + if (currentTimeDiff != 0) { + currentTimeFrameRate = 1000 / currentTimeDiff; + } // System.out.println("XXXX READING PACKET: " + timestamp); From 3676d44d71184773dd1be84c2276b2ce34a21b37 Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Wed, 19 Jun 2024 15:14:06 +0200 Subject: [PATCH 17/33] Fixes to Opus timeframing --- .../restcomm/media/codec/opus/Decoder.java | 2 +- .../media/server/impl/AbstractSource.java | 15 ++++++----- .../media/server/impl/rtp/JitterBuffer.java | 26 +++++++++++++------ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java b/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java index 593c6e233..e78b811c2 100644 --- a/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java +++ b/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java @@ -101,7 +101,7 @@ public Frame process(Frame frame) { res.setOffset(0); res.setLength(frameSize * 2); res.setTimestamp(frame.getTimestamp()); - res.setDuration(frameSize * SAMPLE_LENGTH); + res.setDuration(frameSize * SAMPLE_LENGTH * 1000); res.setSequenceNumber(frame.getSequenceNumber()); res.setEOM(frame.isEOM()); res.setFormat(linear); diff --git a/component/src/main/java/org/mobicents/media/server/impl/AbstractSource.java b/component/src/main/java/org/mobicents/media/server/impl/AbstractSource.java index e67ddcd6a..f0ff44c0d 100644 --- a/component/src/main/java/org/mobicents/media/server/impl/AbstractSource.java +++ b/component/src/main/java/org/mobicents/media/server/impl/AbstractSource.java @@ -390,14 +390,15 @@ public long perform() { readCount++; frame = evolve(timestamp); if (frame == null) { - if(readCount==1) - { - //stop if frame was not generated - isSynchronized = false; - return 0; - } - else +// if(readCount==1) +// { +// //stop if frame was not generated +// isSynchronized = false; +// return 0; +// } +// else { + timestamp += 20000000; //frame was generated so continue scheduler.submit(this,queueNumber); return 0; diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 6ece22f3e..0d22e4644 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -307,14 +307,13 @@ public Frame read(long timestamp) { int size = queue.size(); - if (size < BUFFER_SIZE_MIN) { -// System.out.println("XXXX NULL 1 "); - return null; - } - long currentTime = System.currentTimeMillis(); + + long currentTime = timestamp / 1000000 + 20; long currentTimeDiff = 20; +// System.out.println(this.hashCode() + "--- XXXX CURR time frames " + decodedFrameTime.toString()); + if (!decodedFrameTime.isEmpty()) { currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); } @@ -325,13 +324,23 @@ public Frame read(long timestamp) { currentTimeFrameRate = 1000 / currentTimeDiff; } +// System.out.println(this.hashCode() + "--- XXXX CURR TIME DIFF " + currentTimeDiff + " CT " + currentTime); + + // System.out.println("XXXX READING PACKET: " + timestamp); -// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + + +// System.out.println(this.hashCode() + " -- [ " + System.currentTimeMillis() + "]" + "XXXX READING PACKET: " + size + " " + currentTime + " CDIFF" + currentTimeDiff + " AF: " + avgFrameRate + " CF: " + currentTimeFrameRate + " DFS: " + decodedFrameTime.size()) ; // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + if (size < BUFFER_SIZE_MIN) { +// System.out.println("XXXX NULL 1 "); + return null; + } + if (size < BUFFER_SIZE_NOR) { if (currentTimeFrameRate > avgFrameRate) { // System.out.println("XXXX NULL 2 "); @@ -386,8 +395,9 @@ public Frame read(long timestamp) { lastFrameRate = currentTimeFrameRate; decodedFrameTime.push(currentTime); - avgFrameRate = (currentTime - decodedFrameTime.peekLast()) / decodedFrameTime.size(); - if (avgFrameRate == 0) avgFrameRate = 50; + long frameRateDiff = (currentTime - decodedFrameTime.peekLast()); + if (frameRateDiff == 0) avgFrameRate = 50; + else avgFrameRate = decodedFrameTime.size() * 1000 / frameRateDiff; // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); From c1887af2d821950f634e22286380bee9b9dc2af4 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 15:21:06 +0200 Subject: [PATCH 18/33] decrease buffer size --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 0d22e4644..744eebcbc 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -68,6 +68,8 @@ public class JitterBuffer implements Serializable { private final double SPEED_NOR = 1.5; private final double SPEED_SLOW = 0.3; private final int NUM_FRAME_TIME_HISTORY = 60; + + private final int TARGET_FRAME_RATE = 50; private long avgFrameRate; private double lastFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); @@ -374,8 +376,14 @@ public Frame read(long timestamp) { } + Frame frame = queue.remove(0); + if (size > BUFFER_SIZE_MIN && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { + frame = queue.remove(0); + } + + if (this.dumpConfig != null) { JitterBufferRTPDump dump = rtpDump.get(); if (dump != null) { From 68fa2bde135c239dd90901ce7aa705bb1940b6b0 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 15:42:20 +0200 Subject: [PATCH 19/33] fix peaks --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 744eebcbc..c7981c0ec 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -352,7 +352,7 @@ public Frame read(long timestamp) { } if (size < BUFFER_SIZE_MAX) { - if (currentTimeFrameRate > (SPEED_NOR*avgFrameRate)) { + if (currentTimeFrameRate > (SPEED_NOR*avgFrameRate) && currentTimeFrameRate > (SPEED_NOR*lastFrameRate)) { // System.out.println("XXXX NULL 2 "); return null; } @@ -364,7 +364,7 @@ public Frame read(long timestamp) { } if (size >= BUFFER_SIZE_MAX) { - if (currentTimeFrameRate > SPEED_FAST*avgFrameRate) { + if (currentTimeFrameRate > SPEED_FAST*avgFrameRate && currentTimeFrameRate > (SPEED_NOR*lastFrameRate)) { // System.out.println("XXXX NULL 2 "); return null; } From 440936b86806affa6b5b45879c499c9e2e5e7155 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 15:51:41 +0200 Subject: [PATCH 20/33] keep max --- .../java/org/mobicents/media/server/impl/rtp/JitterBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index c7981c0ec..a84620b4e 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -379,7 +379,7 @@ public Frame read(long timestamp) { Frame frame = queue.remove(0); - if (size > BUFFER_SIZE_MIN && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { + if (size > BUFFER_SIZE_MAX && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { frame = queue.remove(0); } From d2e069e13ec773a953802ee67010d290664d8da6 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 19 Jun 2024 11:02:47 +0200 Subject: [PATCH 21/33] avg is back --- .../media/server/impl/rtp/JitterBuffer.java | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index a84620b4e..f733bc41d 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -70,7 +70,7 @@ public class JitterBuffer implements Serializable { private final int NUM_FRAME_TIME_HISTORY = 60; private final int TARGET_FRAME_RATE = 50; - private long avgFrameRate; + private double avgFrameRate; private double lastFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); @@ -128,7 +128,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.rtpClock = clock; this.scheduler = scheduler; this.lastFrameRate = 1.0f; - this.avgFrameRate = 50L; + this.avgFrameRate = 1.0f; if (dumpDir != null) { this.dumpDir = dumpDir; @@ -320,11 +320,7 @@ public Frame read(long timestamp) { currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); } - long currentTimeFrameRate = 50; - if (currentTimeDiff != 0) { - currentTimeFrameRate = 1000 / currentTimeDiff; - } // System.out.println(this.hashCode() + "--- XXXX CURR TIME DIFF " + currentTimeDiff + " CT " + currentTime); @@ -343,8 +339,16 @@ public Frame read(long timestamp) { return null; } + +// System.out.println("XXXX READING PACKET: " + timestamp); + +// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + + +// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); + if (size < BUFFER_SIZE_NOR) { - if (currentTimeFrameRate > avgFrameRate) { + if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { // System.out.println("XXXX NULL 2 "); return null; } @@ -352,8 +356,9 @@ public Frame read(long timestamp) { } if (size < BUFFER_SIZE_MAX) { - if (currentTimeFrameRate > (SPEED_NOR*avgFrameRate) && currentTimeFrameRate > (SPEED_NOR*lastFrameRate)) { -// System.out.println("XXXX NULL 2 "); + if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && + currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { +// System.out.println("XXXX NULL 3 "); return null; } // if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && @@ -364,8 +369,10 @@ public Frame read(long timestamp) { } if (size >= BUFFER_SIZE_MAX) { - if (currentTimeFrameRate > SPEED_FAST*avgFrameRate && currentTimeFrameRate > (SPEED_NOR*lastFrameRate)) { -// System.out.println("XXXX NULL 2 "); + + if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && + currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { +// System.out.println("XXXX NULL 4 "); return null; } // if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && @@ -379,9 +386,9 @@ public Frame read(long timestamp) { Frame frame = queue.remove(0); - if (size > BUFFER_SIZE_MAX && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { - frame = queue.remove(0); - } +// if (size > BUFFER_SIZE_MAX && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { +// frame = queue.remove(0); +// } if (this.dumpConfig != null) { @@ -401,10 +408,10 @@ public Frame read(long timestamp) { frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); - lastFrameRate = currentTimeFrameRate; + lastFrameRate = 1000.0 / currentTimeDiff; decodedFrameTime.push(currentTime); long frameRateDiff = (currentTime - decodedFrameTime.peekLast()); - if (frameRateDiff == 0) avgFrameRate = 50; + if (frameRateDiff == 0) avgFrameRate = 1.0f; else avgFrameRate = decodedFrameTime.size() * 1000 / frameRateDiff; // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -453,7 +460,7 @@ public void restart() { isn=-1; lastFrameRate = 1.0f; - avgFrameRate = 50L; + avgFrameRate = 1.0f; decodedFrameTime.clear(); From fe09497c067c7b996f3cab2445c0bec0146e9a2b Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Thu, 20 Jun 2024 14:37:11 +0200 Subject: [PATCH 22/33] avg is now int --- .../media/server/impl/rtp/JitterBuffer.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index f733bc41d..8102f7be6 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -70,7 +70,7 @@ public class JitterBuffer implements Serializable { private final int NUM_FRAME_TIME_HISTORY = 60; private final int TARGET_FRAME_RATE = 50; - private double avgFrameRate; + private int avgFrameRate; private double lastFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); @@ -128,7 +128,7 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler this.rtpClock = clock; this.scheduler = scheduler; this.lastFrameRate = 1.0f; - this.avgFrameRate = 1.0f; + this.avgFrameRate = 50; if (dumpDir != null) { this.dumpDir = dumpDir; @@ -308,9 +308,6 @@ public Frame read(long timestamp) { int size = queue.size(); - - - long currentTime = timestamp / 1000000 + 20; long currentTimeDiff = 20; @@ -411,8 +408,8 @@ public Frame read(long timestamp) { lastFrameRate = 1000.0 / currentTimeDiff; decodedFrameTime.push(currentTime); long frameRateDiff = (currentTime - decodedFrameTime.peekLast()); - if (frameRateDiff == 0) avgFrameRate = 1.0f; - else avgFrameRate = decodedFrameTime.size() * 1000 / frameRateDiff; + if (frameRateDiff == 0) avgFrameRate = 50; + else avgFrameRate = (int) (decodedFrameTime.size() * 1000 / frameRateDiff); // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); @@ -460,7 +457,7 @@ public void restart() { isn=-1; lastFrameRate = 1.0f; - avgFrameRate = 1.0f; + avgFrameRate = 50; decodedFrameTime.clear(); From 6e435d1ca91ef792d2906888450e6bbc642f81c1 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Thu, 20 Jun 2024 15:26:51 +0200 Subject: [PATCH 23/33] if else... --- .../media/server/impl/rtp/JitterBuffer.java | 23 +--- .../server/impl/rtp/JitterBufferTest.java | 106 +++++++++--------- 2 files changed, 59 insertions(+), 70 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 8102f7be6..a3cef58b5 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -139,17 +139,6 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler } } - /** - * - * @return the current value of the network RTP jitter. The value is in normalized form as specified in RFC 3550 - * http://tools.ietf.org/html/rfc3550#appendix-A.8 - */ - public long getEstimatedJitter() { - long jitterEstimate = currentJitter >> 4; - // logger.info(String.format("Jitter estimated at %d. Current transit time is %d.", jitterEstimate, currentTransit)); - return jitterEstimate; - } - /** * Get the number of dropped packets. * @@ -311,7 +300,7 @@ public Frame read(long timestamp) { long currentTime = timestamp / 1000000 + 20; long currentTimeDiff = 20; -// System.out.println(this.hashCode() + "--- XXXX CURR time frames " + decodedFrameTime.toString()); + System.out.println(this.hashCode() + "CURR time frames " + decodedFrameTime.toString()); if (!decodedFrameTime.isEmpty()) { currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); @@ -319,10 +308,10 @@ public Frame read(long timestamp) { -// System.out.println(this.hashCode() + "--- XXXX CURR TIME DIFF " + currentTimeDiff + " CT " + currentTime); + System.out.println("CURR TIME DIFF " + currentTimeDiff + " CT " + currentTime); -// System.out.println("XXXX READING PACKET: " + timestamp); + System.out.println("READING PACKET: " + timestamp); @@ -344,7 +333,7 @@ public Frame read(long timestamp) { // System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - if (size < BUFFER_SIZE_NOR) { + else if (size < BUFFER_SIZE_NOR) { if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { // System.out.println("XXXX NULL 2 "); return null; @@ -352,7 +341,7 @@ public Frame read(long timestamp) { } - if (size < BUFFER_SIZE_MAX) { + else if (size < BUFFER_SIZE_MAX) { if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { // System.out.println("XXXX NULL 3 "); @@ -365,7 +354,7 @@ public Frame read(long timestamp) { // } } - if (size >= BUFFER_SIZE_MAX) { + else if (size >= BUFFER_SIZE_MAX) { if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index 9e07244e5..95d3299be 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -187,58 +187,58 @@ public void testOverflow() { assertEquals(1, data.getSequenceNumber()); } - @Test - /** - * - * Test that network jitter for RTP packets is estimated correctly - * - * http://tools.ietf.org/html/rfc3550#appendix-A.8 - */ - public void testJitter() { - // the timestamp for each packet increases by 10ms=160 timestamp units for sampling rate 8KHz - RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); - RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); - RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 3, 123, new byte[160], 0, 160); - RtpPacket p4 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 4, 123, new byte[160], 0, 160); - RtpPacket p5 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 5, 123, new byte[160], 0, 160); - - - long jitterDeltaLimit = 1; // 1 sampling units delta for timing and rounding errors , i.e. 1/8ms - - //write first packet, expected jitter = 0 - jitterBuffer.write(p1,AVProfile.audio.find(8)); - assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); - - // move time forward by 20ms and write the second packet - // the transit time should remain approximately the same - near 0ms. - // expected jitter = 0; - wallClock.tick(20000000L); - jitterBuffer.write(p2,AVProfile.audio.find(8)); - assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); - - // move time forward by 30ms and write the next packet - // the transit time should increase by 10ms, - // as suggested by the difference in the third packet timestamp (160*3) and the 20ms delay for the server to receive the second packet - // expected jitter should be close to the 10ms delay in timestamp units/16, i.e. 80/16. - wallClock.tick(30000000L); - jitterBuffer.write(p3,AVProfile.audio.find(8)); - assertEquals(5, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); - - //move time forward by 20ms and write the next packet - //the transit time does not change from the previous packet. - // The jitter should stay approximately the same. - wallClock.tick(20000000L); - jitterBuffer.write(p4,AVProfile.audio.find(8)); - assertEquals(4, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); - - //move time forward by 30ms and write the next packet - //packet was delayed 10ms again. - // The estimated jitter should increase significantly, by nearly 5ms (80/16) - wallClock.tick(30000000L); - jitterBuffer.write(p5,AVProfile.audio.find(8)); - assertEquals(9, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); - - } +// @Test +// /** +// * +// * Test that network jitter for RTP packets is estimated correctly +// * +// * http://tools.ietf.org/html/rfc3550#appendix-A.8 +// */ +// public void testJitter() { +// // the timestamp for each packet increases by 10ms=160 timestamp units for sampling rate 8KHz +// RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); +// RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); +// RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 3, 123, new byte[160], 0, 160); +// RtpPacket p4 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 4, 123, new byte[160], 0, 160); +// RtpPacket p5 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 5, 123, new byte[160], 0, 160); +// +// +// long jitterDeltaLimit = 1; // 1 sampling units delta for timing and rounding errors , i.e. 1/8ms +// +// //write first packet, expected jitter = 0 +// jitterBuffer.write(p1,AVProfile.audio.find(8)); +// assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); +// +// // move time forward by 20ms and write the second packet +// // the transit time should remain approximately the same - near 0ms. +// // expected jitter = 0; +// wallClock.tick(20000000L); +// jitterBuffer.write(p2,AVProfile.audio.find(8)); +// assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); +// +// // move time forward by 30ms and write the next packet +// // the transit time should increase by 10ms, +// // as suggested by the difference in the third packet timestamp (160*3) and the 20ms delay for the server to receive the second packet +// // expected jitter should be close to the 10ms delay in timestamp units/16, i.e. 80/16. +// wallClock.tick(30000000L); +// jitterBuffer.write(p3,AVProfile.audio.find(8)); +// assertEquals(5, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); +// +// //move time forward by 20ms and write the next packet +// //the transit time does not change from the previous packet. +// // The jitter should stay approximately the same. +// wallClock.tick(20000000L); +// jitterBuffer.write(p4,AVProfile.audio.find(8)); +// assertEquals(4, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); +// +// //move time forward by 30ms and write the next packet +// //packet was delayed 10ms again. +// // The estimated jitter should increase significantly, by nearly 5ms (80/16) +// wallClock.tick(30000000L); +// jitterBuffer.write(p5,AVProfile.audio.find(8)); +// assertEquals(9, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); +// +// } private RtpPacket[] createStream(int size) { RtpPacket[] stream = new RtpPacket[size]; @@ -258,7 +258,7 @@ private void checkSequence(Frame[] media) throws Exception { throw new Exception("Null data at position: " + i); } - if (media[i + 1] == null) { + if (media[i + 1] == null) { throw new Exception("Null data at position: " + (i+1)); } From da5d6773166e04d7dd03885c0236ce3c5126597c Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Fri, 21 Jun 2024 11:19:02 +0200 Subject: [PATCH 24/33] cleanup --- .../restcomm/media/codec/opus/Decoder.java | 4 +- .../media/server/impl/rtp/BufferListener.java | 31 --- .../media/server/impl/rtp/JitterBuffer.java | 124 ++------- .../media/server/impl/rtp/RTPInput.java | 22 +- .../media/server/impl/rtp/RtpHandler.java | 7 +- .../server/impl/rtp/JitterBufferTest.java | 248 ++++++++---------- 6 files changed, 134 insertions(+), 302 deletions(-) delete mode 100644 io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/BufferListener.java diff --git a/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java b/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java index e78b811c2..5ffc62aa0 100644 --- a/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java +++ b/codecs/opus/src/main/java/org/restcomm/media/codec/opus/Decoder.java @@ -52,7 +52,7 @@ public class Decoder implements Codec { private final int OPUS_SAMPLE_RATE = 8000; private final int MAX_FRAME_SIZE = 160; - private final int SAMPLE_LENGTH = 1000000 / OPUS_SAMPLE_RATE; // 1s / 8Khz ~ 125000ns / sample + private final int SAMPLE_LENGTH = 1000 / OPUS_SAMPLE_RATE; // 1s / 8Khz ~ 125000ns / sample private short[] decodedBuff = new short[MAX_FRAME_SIZE]; @@ -101,7 +101,7 @@ public Frame process(Frame frame) { res.setOffset(0); res.setLength(frameSize * 2); res.setTimestamp(frame.getTimestamp()); - res.setDuration(frameSize * SAMPLE_LENGTH * 1000); + res.setDuration(frameSize * SAMPLE_LENGTH); res.setSequenceNumber(frame.getSequenceNumber()); res.setEOM(frame.isEOM()); res.setFormat(linear); diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/BufferListener.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/BufferListener.java deleted file mode 100644 index 8ad29df0c..000000000 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/BufferListener.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * JBoss, Home of Professional Open Source - * Copyright 2011, Red Hat, Inc. and individual contributors - * by the @authors tag. See the copyright.txt in the distribution for a - * full listing of individual contributors. - * - * This is free software; you can redistribute it and/or modify it - * under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation; either version 2.1 of - * the License, or (at your option) any later version. - * - * This software is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this software; if not, write to the Free - * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA - * 02110-1301 USA, or see the FSF site: http://www.fsf.org. - */ - -package org.mobicents.media.server.impl.rtp; - -/** - * - * @author kulikov - */ -public interface BufferListener { - public void onFill(); -} diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index a3cef58b5..9b37b4996 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -64,14 +64,13 @@ public class JitterBuffer implements Serializable { private final int BUFFER_SIZE_NOR = 3; private final int BUFFER_SIZE_MIN = 1; - private final double SPEED_FAST = 2; - private final double SPEED_NOR = 1.5; - private final double SPEED_SLOW = 0.3; + private final double SPEED_FAST = 0.3; + private final double SPEED_NOR = 0.8; + private final double SPEED_SLOW = 1.0; private final int NUM_FRAME_TIME_HISTORY = 60; - private final int TARGET_FRAME_RATE = 50; private int avgFrameRate; - private double lastFrameRate; + private int lastFrameRate; private LinkedList decodedFrameTime = new LinkedList<>(); //The underlying buffer size @@ -88,21 +87,9 @@ public class JitterBuffer implements Serializable { //initial value equals to infinity private long arrivalDeadLine = 0; - - //The number of dropped packets - private int dropCount; - //known duration of media wich contains in this buffer. private volatile long duration; - //buffer's monitor - private BufferListener listener; - - /** - * continuously updated value of network jitter - */ - private long currentJitter = 0; - //currently used format private RTPFormat format; @@ -127,7 +114,7 @@ public class JitterBuffer implements Serializable { public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler scheduler, Path dumpDir) { this.rtpClock = clock; this.scheduler = scheduler; - this.lastFrameRate = 1.0f; + this.lastFrameRate = 50; this.avgFrameRate = 50; if (dumpDir != null) { @@ -139,29 +126,11 @@ public JitterBuffer(RtpClock clock, int jitterBufferSize, PriorityQueueScheduler } } - /** - * Get the number of dropped packets. - * - * @return the number of dropped packets. - */ - public int getDropped() { - return dropCount; - } - public void setBufferInUse(boolean useBuffer) { this.useBuffer=useBuffer; } - /** - * Assigns listener for this buffer. - * - * @param listener the listener object. - */ - public void setListener(BufferListener listener) { - this.listener = listener; - } - /** * Accepts specified packet * @@ -176,10 +145,6 @@ public void write(RtpPacket packet, RTPFormat format) { return; } - -// System.out.println("XXXX WRITING PACKET: "); - - if (this.format == null || this.format.getID() != format.getID()) { logger.info( "Format has been changed: " + @@ -261,7 +226,6 @@ public void write(RtpPacket packet, RTPFormat format) { if (duration < 0 && queue.size() > 1) { logger.warn("Something messy happened. Reseting jitter buffer!"); reset(); - return; } } finally { @@ -269,6 +233,10 @@ public void write(RtpPacket packet, RTPFormat format) { } } + public int getBufferSize() { + return queue.size(); + } + /** * Polls packet from buffer's head. * @@ -300,83 +268,38 @@ public Frame read(long timestamp) { long currentTime = timestamp / 1000000 + 20; long currentTimeDiff = 20; - System.out.println(this.hashCode() + "CURR time frames " + decodedFrameTime.toString()); - if (!decodedFrameTime.isEmpty()) { currentTimeDiff = currentTime - decodedFrameTime.peekFirst(); } - - - System.out.println("CURR TIME DIFF " + currentTimeDiff + " CT " + currentTime); - - - System.out.println("READING PACKET: " + timestamp); - - - -// System.out.println(this.hashCode() + " -- [ " + System.currentTimeMillis() + "]" + "XXXX READING PACKET: " + size + " " + currentTime + " CDIFF" + currentTimeDiff + " AF: " + avgFrameRate + " CF: " + currentTimeFrameRate + " DFS: " + decodedFrameTime.size()) ; - - -// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - if (size < BUFFER_SIZE_MIN) { -// System.out.println("XXXX NULL 1 "); + System.out.println("SKIP MIN"); return null; } - -// System.out.println("XXXX READING PACKET: " + timestamp); - -// System.out.println(this.hashCode() + "-- XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - - -// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + decodedFrameTime.peekFirst() + " " + (currentTime - decodedFrameTime.peekFirst()) + " " + avgFrameRate + " " + lastFrameRate); - else if (size < BUFFER_SIZE_NOR) { if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { -// System.out.println("XXXX NULL 2 "); + System.out.println("SKIP NOR: " + currentTimeDiff + " " + (1000 * SPEED_SLOW / avgFrameRate)); return null; } - } - - else if (size < BUFFER_SIZE_MAX) { + } else if (size < BUFFER_SIZE_MAX) { if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { -// System.out.println("XXXX NULL 3 "); + System.out.println("SKIP < MAX: " + currentTimeDiff + " " + (1000 * SPEED_NOR / avgFrameRate) + " " + (1000 * SPEED_NOR / lastFrameRate)); return null; } -// if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && -// currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { -//// System.out.println("XXXX NULL 3 "); -// return null; -// } - } - - else if (size >= BUFFER_SIZE_MAX) { - + } else { if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { -// System.out.println("XXXX NULL 4 "); + System.out.println("SKIP >= MAX: " + currentTimeDiff + " " + (1000 * SPEED_FAST / avgFrameRate) + " " + (1000 * SPEED_FAST / lastFrameRate)); return null; } -// if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && -// currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { -//// System.out.println("XXXX NULL 4 "); -// return null; -// } - } - + } Frame frame = queue.remove(0); -// if (size > BUFFER_SIZE_MAX && avgFrameRate == TARGET_FRAME_RATE && currentTimeFrameRate == TARGET_FRAME_RATE && (currentTime % 1000) == 0) { -// frame = queue.remove(0); -// } - - if (this.dumpConfig != null) { JitterBufferRTPDump dump = rtpDump.get(); if (dump != null) { @@ -385,21 +308,21 @@ else if (size >= BUFFER_SIZE_MAX) { } } -// System.out.println("XXXX READING AND WILL RETURN: "); - - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); //convert duration to nanoseconds frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); - lastFrameRate = 1000.0 / currentTimeDiff; + lastFrameRate = (int) (1000.0 / currentTimeDiff); decodedFrameTime.push(currentTime); long frameRateDiff = (currentTime - decodedFrameTime.peekLast()); + + int dftSize = decodedFrameTime.size()-1 ; + if (dftSize == 0) dftSize = 1; + if (frameRateDiff == 0) avgFrameRate = 50; - else avgFrameRate = (int) (decodedFrameTime.size() * 1000 / frameRateDiff); -// System.out.println("XXXX READING PACKET: " + size + " " + currentTime + " " + avgFrameRate + " " + lastFrameRate); + else avgFrameRate = (int) (dftSize * 1000 / frameRateDiff); if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) { @@ -441,11 +364,10 @@ private void restartRecording() { public void restart() { reset(); arrivalDeadLine = 0; - dropCount=0; format=null; isn=-1; - lastFrameRate = 1.0f; + lastFrameRate = 50; avgFrameRate = 50; decodedFrameTime.clear(); diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RTPInput.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RTPInput.java index cc11ab34e..d84edde8a 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RTPInput.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RTPInput.java @@ -40,7 +40,7 @@ * * The Media source of RTP data. */ -public class RTPInput extends AbstractSource implements BufferListener { +public class RTPInput extends AbstractSource { private static final long serialVersionUID = -737259897530641186L; @@ -53,10 +53,7 @@ public class RTPInput extends AbstractSource implements BufferListener { //digital signaling processor private Processor dsp; - - protected Integer preEvolveCount=0; - protected Integer evolveCount=0; - + private static final Logger logger = org.apache.logging.log4j.LogManager.getLogger(RTPInput.class); private AudioInput input; @@ -100,10 +97,6 @@ public Processor getDsp() { return this.dsp; } - protected int getPacketsLost() { - return 0; - } - @Override public Frame evolve(long timestamp) { Frame currFrame=rxBuffer.read(timestamp); @@ -127,14 +120,5 @@ public Frame evolve(long timestamp) { return currFrame; } - - /** - * RX buffer's call back method. - * - * This method is called when rxBuffer is full and it is time to start - * transmission to the consumer. - */ - public void onFill() { - this.wakeup(); - } + } \ No newline at end of file diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java index 553cfd049..98106f44c 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/RtpHandler.java @@ -77,7 +77,6 @@ public RtpHandler(PriorityQueueScheduler scheduler, RtpClock clock, RtpClock oob this.jitterBuffer = new JitterBuffer(this.rtpClock, this.jitterBufferSize, scheduler, dumpDir); this.rtpInput = new RTPInput(scheduler, jitterBuffer); - this.jitterBuffer.setListener(this.rtpInput); this.dtmfInput = new DtmfInput(scheduler, oobClock); this.rtpFormats = new RTPFormats(); @@ -103,11 +102,7 @@ public RTPInput getRtpInput() { public DtmfInput getDtmfInput() { return dtmfInput; } - - public boolean isLoopable() { - return loopable; - } - + public void setLoopable(boolean loopable) { this.loopable = loopable; } diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index 95d3299be..b922398ec 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -23,7 +23,9 @@ package org.mobicents.media.server.impl.rtp; import java.net.InetSocketAddress; +import java.util.LinkedList; import java.util.Random; +import java.util.HashMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -72,174 +74,89 @@ public void setUp() { public void tearDown() { } - @Test - public void testNormalReadWrite() throws Exception { + public void failingOutOforder() throws Exception { RtpPacket[] stream = createStream(100); + HashMap> packets = reorderWithDelay(10, 10, stream); Frame[] media = new Frame[stream.length]; + int[] bufferSize = new int[stream.length]; for (int i = 0; i < stream.length; i++) { + if (packets.containsKey(i)) { + for (RtpPacket rtpPacket : packets.get(i)) { + System.out.println("Packet: " + rtpPacket.getSeqNumber()); + jitterBuffer.write(rtpPacket, AVProfile.audio.find(8)); + } + } + wallClock.tick(20000000L); - jitterBuffer.write(stream[i],AVProfile.audio.find(8)); media[i] = jitterBuffer.read(wallClock.getTime()); + bufferSize[i] = jitterBuffer.getBufferSize(); } +// this.checkMaxBufferSize(bufferSize, 9); this.checkSequence(media); assertEquals(0, 0); } @Test - public void testInnerSort() throws Exception { - // todo fix - RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); - RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); - RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 3, 123, new byte[160], 0, 160); - RtpPacket p4 = RtpPacket.outgoing(local,remote,false, 8, 4, 160 * 4, 123, new byte[160], 0, 160); - RtpPacket p5 = RtpPacket.outgoing(local,remote,false, 8, 5, 160 * 5, 123, new byte[160], 0, 160); - - jitterBuffer.write(p1,AVProfile.audio.find(8)); - jitterBuffer.write(p2,AVProfile.audio.find(8)); - jitterBuffer.write(p4,AVProfile.audio.find(8)); - jitterBuffer.write(p3,AVProfile.audio.find(8)); - - Frame buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(1, buffer.getSequenceNumber()); - - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(2, buffer.getSequenceNumber()); - - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(3, buffer.getSequenceNumber()); + public void testBuffering() throws Exception { + RtpPacket[] stream = createStream(1000); - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(4, buffer.getSequenceNumber()); + Frame[] media = new Frame[stream.length]; + int[] bufferSize = new int[stream.length]; + for (int i = 0; i < stream.length; i++) { + wallClock.tick(20000000L); + if (i%5 == 0) { + jitterBuffer.write(stream[i], AVProfile.audio.find(8)); + jitterBuffer.write(stream[i+1], AVProfile.audio.find(8)); + jitterBuffer.write(stream[i+2], AVProfile.audio.find(8)); + jitterBuffer.write(stream[i+3], AVProfile.audio.find(8)); + jitterBuffer.write(stream[i+4], AVProfile.audio.find(8)); + } + media[i] = jitterBuffer.read(wallClock.getTime()); + bufferSize[i] = jitterBuffer.getBufferSize(); + } + this.checkMaxBufferSize(bufferSize, 4); + this.checkSequence(media); + assertEquals(0, 0); } @Test - public void testOutstanding() throws Exception { - RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); - RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); - RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 3, 123, new byte[160], 0, 160); - RtpPacket p4 = RtpPacket.outgoing(local,remote,false, 8, 4, 160 * 4, 123, new byte[160], 0, 160); - RtpPacket p5 =RtpPacket.outgoing(local,remote,false, 8, 5, 160 * 5, 123, new byte[160], 0, 160); - - jitterBuffer.write(p1,AVProfile.audio.find(8)); - jitterBuffer.write(p3,AVProfile.audio.find(8)); - jitterBuffer.write(p5,AVProfile.audio.find(8)); - - assertEquals(0, jitterBuffer.getDropped()); - - //60ms + 40ms - wallClock.tick(100000000L); - - Frame buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(1, buffer.getSequenceNumber()); - - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(3, buffer.getSequenceNumber()); - - jitterBuffer.write(p2,AVProfile.audio.find(8)); - assertEquals(1, jitterBuffer.getDropped()); - - - -// buffer = jitterBuffer.read(wallClock.getTime()); -// assertEquals(3, buffer.getSequenceNumber()); + public void testNormalReadWrite() throws Exception { + RtpPacket[] stream = createStream(1000); -// buffer = jitterBuffer.read(wallClock.getTime()); -// assertEquals(null, buffer); + Frame[] media = new Frame[stream.length]; + for (int i = 0; i < stream.length; i++) { + wallClock.tick(20000000L); + jitterBuffer.write(stream[i], AVProfile.audio.find(8)); + media[i] = jitterBuffer.read(wallClock.getTime()); + } + this.checkSequence(media); + assertEquals(0, 0); } @Test - public void testEmpty() throws Exception { - RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); //new RtpPacket(172, false); - RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); //new RtpPacket(172, false); - RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 3, 123, new byte[160], 0, 160); //new RtpPacket(172, false); - - jitterBuffer.write(p1,AVProfile.audio.find(8)); - jitterBuffer.write(p2,AVProfile.audio.find(8)); - jitterBuffer.write(p3,AVProfile.audio.find(8)); - - Frame buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(1, buffer.getSequenceNumber()); - - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(2, buffer.getSequenceNumber()); + public void testOrdering() throws Exception { + RtpPacket[] stream = createStream(1000); + shuffle(stream); - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(3, buffer.getSequenceNumber()); - - buffer = jitterBuffer.read(wallClock.getTime()); - assertEquals(null, buffer); - - } + for (RtpPacket rtpPacket : stream) { + jitterBuffer.write(rtpPacket, AVProfile.audio.find(8)); + } - @Test - public void testOverflow() { - RtpPacket[] stream = createStream(5); + Frame[] media = new Frame[stream.length]; for (int i = 0; i < stream.length; i++) { - jitterBuffer.write(stream[i],AVProfile.audio.find(8)); + wallClock.tick(20000000L); + media[i] = jitterBuffer.read(wallClock.getTime()); } - Frame data = jitterBuffer.read(wallClock.getTime()); - assertEquals(1, data.getSequenceNumber()); + this.checkSequence(media); + assertEquals(0, 0); } -// @Test -// /** -// * -// * Test that network jitter for RTP packets is estimated correctly -// * -// * http://tools.ietf.org/html/rfc3550#appendix-A.8 -// */ -// public void testJitter() { -// // the timestamp for each packet increases by 10ms=160 timestamp units for sampling rate 8KHz -// RtpPacket p1 = RtpPacket.outgoing(local,remote,false, 8, 1, 160 * 1, 123, new byte[160], 0, 160); -// RtpPacket p2 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 2, 123, new byte[160], 0, 160); -// RtpPacket p3 = RtpPacket.outgoing(local,remote,false, 8, 2, 160 * 3, 123, new byte[160], 0, 160); -// RtpPacket p4 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 4, 123, new byte[160], 0, 160); -// RtpPacket p5 = RtpPacket.outgoing(local,remote,false, 8, 3, 160 * 5, 123, new byte[160], 0, 160); -// -// -// long jitterDeltaLimit = 1; // 1 sampling units delta for timing and rounding errors , i.e. 1/8ms -// -// //write first packet, expected jitter = 0 -// jitterBuffer.write(p1,AVProfile.audio.find(8)); -// assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); -// -// // move time forward by 20ms and write the second packet -// // the transit time should remain approximately the same - near 0ms. -// // expected jitter = 0; -// wallClock.tick(20000000L); -// jitterBuffer.write(p2,AVProfile.audio.find(8)); -// assertEquals(0, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); -// -// // move time forward by 30ms and write the next packet -// // the transit time should increase by 10ms, -// // as suggested by the difference in the third packet timestamp (160*3) and the 20ms delay for the server to receive the second packet -// // expected jitter should be close to the 10ms delay in timestamp units/16, i.e. 80/16. -// wallClock.tick(30000000L); -// jitterBuffer.write(p3,AVProfile.audio.find(8)); -// assertEquals(5, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); -// -// //move time forward by 20ms and write the next packet -// //the transit time does not change from the previous packet. -// // The jitter should stay approximately the same. -// wallClock.tick(20000000L); -// jitterBuffer.write(p4,AVProfile.audio.find(8)); -// assertEquals(4, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); -// -// //move time forward by 30ms and write the next packet -// //packet was delayed 10ms again. -// // The estimated jitter should increase significantly, by nearly 5ms (80/16) -// wallClock.tick(30000000L); -// jitterBuffer.write(p5,AVProfile.audio.find(8)); -// assertEquals(9, jitterBuffer.getEstimatedJitter(), jitterDeltaLimit); -// -// } - private RtpPacket[] createStream(int size) { RtpPacket[] stream = new RtpPacket[size]; @@ -251,31 +168,76 @@ private RtpPacket[] createStream(int size) { return stream; } + private void checkMaxBufferSize(int[] buffer, int maxSize) throws Exception { + for (int j : buffer) { + assertTrue("Max buffer size exceeded " + j + " > " + maxSize, j <= maxSize); + } + } + private void checkSequence(Frame[] media) throws Exception { + int loss = 0; boolean res = true; for (int i = 0; i < media.length - 1; i++) { - if (media[i] == null) { - throw new Exception("Null data at position: " + i); + if (media[i] == null) { + loss++; + continue; } if (media[i + 1] == null) { - throw new Exception("Null data at position: " + (i+1)); + continue; } res &= (media[i + 1].getSequenceNumber() - media[i].getSequenceNumber() == 1); } + System.out.println("Loss: " + ((100 * loss) / media.length)); + int lossPercent = (100 * loss) / media.length; + assertTrue("Loss is too high " + lossPercent, lossPercent < 10); assertTrue("Wrong sequence ", res); } private void shuffle(RtpPacket[] stream) { Random rnd = new Random(); - for (int k = 0; k < 5; k++) { + for (int k = 0; k < stream.length; k++) { int i = rnd.nextInt(stream.length - 1); + int j = rnd.nextInt(stream.length - 1); RtpPacket tmp = stream[i]; - stream[i] = stream[i + 1]; - stream[i + 1] = tmp; + stream[i] = stream[j]; + stream[j] = tmp; + } + } + + private HashMap> reorderWithDelay(int delay, int jitter, RtpPacket[] stream) { + HashMap> result = new HashMap>(); + + Random rnd = new Random(); + + for (int i = 0; i < stream.length; i++) { + int key = i + delay; + if (jitter > 0) { + if (rnd.nextBoolean()) { + key += rnd.nextInt(jitter); + } else { + key -= rnd.nextInt(jitter); + } + + if (key < 0) key = 0; + } + + + LinkedList list; + if (!result.containsKey(key)) { + list = new LinkedList(); + } else { + list = result.get(key); + } + + list.push(stream[i]); + result.put(key, list); } + + return result; } + } From 3108edc337104c87833ec78a262aeb903bb59cac Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Fri, 21 Jun 2024 13:49:53 +0200 Subject: [PATCH 25/33] drop some packets --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 9b37b4996..4207dc343 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -66,7 +66,7 @@ public class JitterBuffer implements Serializable { private final double SPEED_FAST = 0.3; private final double SPEED_NOR = 0.8; - private final double SPEED_SLOW = 1.0; + private final double SPEED_SLOW = 1.5; private final int NUM_FRAME_TIME_HISTORY = 60; private int avgFrameRate; @@ -296,6 +296,10 @@ else if (size < BUFFER_SIZE_NOR) { return null; } + if (avgFrameRate > 49 && (currentTime % 200) == 0) { + queue.remove(0); + } + } Frame frame = queue.remove(0); From 1c0020dbecc49d59cef94b676af4a2a1cad38303 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 9 Jul 2024 12:03:46 +0200 Subject: [PATCH 26/33] dont println --- .../org/mobicents/media/server/impl/rtp/JitterBuffer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 4207dc343..896a77b33 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -273,26 +273,26 @@ public Frame read(long timestamp) { } if (size < BUFFER_SIZE_MIN) { - System.out.println("SKIP MIN"); +// System.out.println("SKIP MIN"); return null; } else if (size < BUFFER_SIZE_NOR) { if (currentTimeDiff < (1000 * SPEED_SLOW / avgFrameRate)) { - System.out.println("SKIP NOR: " + currentTimeDiff + " " + (1000 * SPEED_SLOW / avgFrameRate)); +// System.out.println("SKIP NOR: " + currentTimeDiff + " " + (1000 * SPEED_SLOW / avgFrameRate)); return null; } } else if (size < BUFFER_SIZE_MAX) { if (currentTimeDiff < (1000 * SPEED_NOR / avgFrameRate) && currentTimeDiff < (1000 * SPEED_NOR / lastFrameRate)) { - System.out.println("SKIP < MAX: " + currentTimeDiff + " " + (1000 * SPEED_NOR / avgFrameRate) + " " + (1000 * SPEED_NOR / lastFrameRate)); +// System.out.println("SKIP < MAX: " + currentTimeDiff + " " + (1000 * SPEED_NOR / avgFrameRate) + " " + (1000 * SPEED_NOR / lastFrameRate)); return null; } } else { if (currentTimeDiff < (1000 * SPEED_FAST / avgFrameRate) && currentTimeDiff < (1000 * SPEED_FAST / lastFrameRate)) { - System.out.println("SKIP >= MAX: " + currentTimeDiff + " " + (1000 * SPEED_FAST / avgFrameRate) + " " + (1000 * SPEED_FAST / lastFrameRate)); +// System.out.println("SKIP >= MAX: " + currentTimeDiff + " " + (1000 * SPEED_FAST / avgFrameRate) + " " + (1000 * SPEED_FAST / lastFrameRate)); return null; } From 3d9e85745f5cf24a3187947a6956ea9669d7ee50 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 9 Jul 2024 21:24:49 +0200 Subject: [PATCH 27/33] fix change in frame timestamp --- .../media/server/impl/rtp/JitterBuffer.java | 40 +++++++++++-------- .../server/impl/rtp/JitterBufferTest.java | 12 ++++-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 896a77b33..eb1c0a6fe 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -168,22 +168,6 @@ public void write(RtpPacket packet, RTPFormat format) { isn = packet.getSeqNumber(); } - // drop outstanding packets - // packet is outstanding if its timestamp of arrived packet is less - // then consumer media time - if (packet.getTimestamp() < this.arrivalDeadLine) { - logger.warn( - "drop packet: dead line=" + arrivalDeadLine + - ", packet time=" + packet.getTimestamp() + - ", seq=" + packet.getSeqNumber() + - ", payload length=" + packet.getPayloadLength() + - ", format=" + this.format.toString() + - ", csrc: " + packet.getContributingSource() - ); - - return; - } - Frame f = packet.toFrame(rtpClock, this.format); f.setDuration(rtpClock.convertToAbsoluteTime(f.getLength())); @@ -212,6 +196,30 @@ public void write(RtpPacket packet, RTPFormat format) { return; } + if (currIndex == -1 && !queue.isEmpty()) { + // drop outstanding packets + // packet is outstanding if its timestamp of arrived packet is less + // then consumer media time + long arrivalDiff = this.arrivalDeadLine - packet.getTimestamp(); + int maxDiff = packet.getPayloadLength() * 50; //1 second + if (arrivalDiff < maxDiff) { + System.out.println( + "drop packet: dead line=" + arrivalDeadLine + + ", packet time=" + packet.getTimestamp() + + ", seq=" + packet.getSeqNumber() + + ", payload length=" + packet.getPayloadLength() + + ", format=" + this.format.toString() + + ", csrc: " + packet.getContributingSource() + + ", arrivalDiff: " + arrivalDiff + + ", maxDiff: " + maxDiff + ); + + return; + } else if (arrivalDiff > 0) { + currIndex = queue.size() - 1; + } + } + queue.add(currIndex + 1, f); // recalculate duration of each frame in queue and overall duration diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index b922398ec..06be0721a 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -118,7 +118,7 @@ public void testBuffering() throws Exception { bufferSize[i] = jitterBuffer.getBufferSize(); } - this.checkMaxBufferSize(bufferSize, 4); + this.checkMaxBufferSize(bufferSize, 6); this.checkSequence(media); assertEquals(0, 0); } @@ -160,11 +160,15 @@ public void testOrdering() throws Exception { private RtpPacket[] createStream(int size) { RtpPacket[] stream = new RtpPacket[size]; - int it = 12345; - - for (int i = 0; i < stream.length; i++) { + int it = 12345000; + int it2 = 12345; + for (int i = 0; i < stream.length/2; i++) { stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it, 123, new byte[160], 0, 160); } + + for (int i = stream.length/2; i < stream.length; i++) { + stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it2, 123, new byte[160], 0, 160); + } return stream; } From b3309f575873bde804b85a36e645bedbdb23b4e7 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Wed, 10 Jul 2024 10:41:46 +0200 Subject: [PATCH 28/33] fix logger --- .../java/org/mobicents/media/server/impl/rtp/JitterBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index eb1c0a6fe..5b29048ff 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -203,7 +203,7 @@ public void write(RtpPacket packet, RTPFormat format) { long arrivalDiff = this.arrivalDeadLine - packet.getTimestamp(); int maxDiff = packet.getPayloadLength() * 50; //1 second if (arrivalDiff < maxDiff) { - System.out.println( + logger.warn( "drop packet: dead line=" + arrivalDeadLine + ", packet time=" + packet.getTimestamp() + ", seq=" + packet.getSeqNumber() + From 4d5fd99c5407cc1ba39edbb1453b20d36360621c Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Thu, 11 Jul 2024 10:14:28 +0200 Subject: [PATCH 29/33] fix arrival deadline and log change of SSRC --- .../media/server/impl/rtp/JitterBuffer.java | 78 ++++++++++--------- .../server/impl/rtp/JitterBufferTest.java | 19 +++-- 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 5b29048ff..45797511f 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -85,10 +85,8 @@ public class JitterBuffer implements Serializable { //packet arrival dead line measured on RTP clock. //initial value equals to infinity - private long arrivalDeadLine = 0; + private long arrivalDeadLine = -1; - //known duration of media wich contains in this buffer. - private volatile long duration; //currently used format private RTPFormat format; @@ -106,6 +104,8 @@ public class JitterBuffer implements Serializable { private Path dumpDir; private List dumpConfig; + private long syncSource = -1; + /** * Creates new instance of jitter. * @@ -166,6 +166,7 @@ public void write(RtpPacket packet, RTPFormat format) { if (isn == -1) { rtpClock.synchronize(packet.getTimestamp()); isn = packet.getSeqNumber(); + syncSource = packet.getSyncSource(); } Frame f = packet.toFrame(rtpClock, this.format); @@ -191,50 +192,43 @@ public void write(RtpPacket packet, RTPFormat format) { ", seq=" + packet.getSeqNumber() + ", payload length=" + packet.getPayloadLength() + ", format=" + this.format.toString() + - ", csrc: " + packet.getContributingSource() + ", ssrc: " + packet.getSyncSource() ); return; } - if (currIndex == -1 && !queue.isEmpty()) { + if (currIndex == -1 && arrivalDeadLine != -1) { // drop outstanding packets // packet is outstanding if its timestamp of arrived packet is less // then consumer media time - long arrivalDiff = this.arrivalDeadLine - packet.getTimestamp(); + long arrivalDiff = packet.getTimestamp() - this.arrivalDeadLine; int maxDiff = packet.getPayloadLength() * 50; //1 second - if (arrivalDiff < maxDiff) { - logger.warn( - "drop packet: dead line=" + arrivalDeadLine + - ", packet time=" + packet.getTimestamp() + - ", seq=" + packet.getSeqNumber() + - ", payload length=" + packet.getPayloadLength() + - ", format=" + this.format.toString() + - ", csrc: " + packet.getContributingSource() + - ", arrivalDiff: " + arrivalDiff + - ", maxDiff: " + maxDiff - ); - - return; - } else if (arrivalDiff > 0) { - currIndex = queue.size() - 1; + if (arrivalDiff < 0) { + if (Math.abs(arrivalDiff) > maxDiff) { + currIndex = queue.size() - 1; + } else { + logger.warn( + "drop packet: dead line=" + arrivalDeadLine + + ", packet time=" + packet.getTimestamp() + + ", seq=" + packet.getSeqNumber() + + ", payload length=" + packet.getPayloadLength() + + ", format=" + this.format.toString() + + ", ssrc: " + packet.getSyncSource() + + ", arrivalDiff: " + arrivalDiff + + ", maxDiff: " + maxDiff + ); + + return; + } } } - queue.add(currIndex + 1, f); - - // recalculate duration of each frame in queue and overall duration - // since we could insert the frame in the middle of the queue - duration = 0; - if (queue.size() > 1) { - duration = queue.get(queue.size() - 1).getTimestamp() - queue.get(0).getTimestamp(); + if (syncSource != packet.getSyncSource()) { + syncSource = packet.getSyncSource(); + logger.warn("New SyncSource: " + syncSource); } - // if overall duration is negative we have some mess here,try to - // reset - if (duration < 0 && queue.size() > 1) { - logger.warn("Something messy happened. Reseting jitter buffer!"); - reset(); - } + queue.add(currIndex + 1, f); } finally { LOCK.unlock(); @@ -260,7 +254,11 @@ public Frame read(long timestamp) { } else { Frame frame = queue.remove(0); - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + if (queue.isEmpty()) { + arrivalDeadLine = -1; + } else { + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + } //convert duration to nanoseconds frame.setDuration(frame.getDuration() * 1000000L); @@ -320,7 +318,12 @@ else if (size < BUFFER_SIZE_NOR) { } } - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + if (queue.isEmpty()) { + arrivalDeadLine = -1; + } else { + //set arrival deadline for the next frame (in rtp time + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + } //convert duration to nanoseconds frame.setDuration(frame.getDuration() * 1000000L); @@ -375,9 +378,10 @@ private void restartRecording() { public void restart() { reset(); - arrivalDeadLine = 0; + arrivalDeadLine = -1; format=null; isn=-1; + syncSource=-1; lastFrameRate = 50; avgFrameRate = 50; diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index 06be0721a..4167aacd2 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -160,15 +160,22 @@ public void testOrdering() throws Exception { private RtpPacket[] createStream(int size) { RtpPacket[] stream = new RtpPacket[size]; - int it = 12345000; - int it2 = 12345; - for (int i = 0; i < stream.length/2; i++) { - stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it, 123, new byte[160], 0, 160); + int it = 1234500000; + int it2 = 0; + int it3 = 1234560000; + int segment = stream.length/3; + for (int i = 0; i < segment; i++) { + stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it, 123, new byte[160], 0, 160); } - for (int i = stream.length/2; i < stream.length; i++) { - stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it2, 123, new byte[160], 0, 160); + for (int i = segment; i < 2*segment; i++) { + stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it2, 123, new byte[160], 0, 160); } + + for (int i = 2*segment; i < stream.length; i++) { + stream[i] = RtpPacket.outgoing(local,remote,false, 8, i + 1, 160 * (i+1) + it3, 123, new byte[160], 0, 160); + } + return stream; } From 8d6b9d873a94b698e468bd91eb782145e5a9a6c6 Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Thu, 11 Jul 2024 10:17:57 +0200 Subject: [PATCH 30/33] fix log --- .../media/server/impl/rtp/JitterBuffer.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 45797511f..163c3b5bc 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -224,8 +224,12 @@ public void write(RtpPacket packet, RTPFormat format) { } if (syncSource != packet.getSyncSource()) { + logger.warn("New SyncSource: " + packet.getSyncSource() + + ", old SyncSource: " + syncSource + + ", arrivalDeadline: " + arrivalDeadLine + + ", timestamp: " + packet.getTimestamp() + ); syncSource = packet.getSyncSource(); - logger.warn("New SyncSource: " + syncSource); } queue.add(currIndex + 1, f); @@ -250,15 +254,13 @@ public Frame read(long timestamp) { LOCK.lock(); if (!useBuffer) { if (queue.isEmpty()) { + arrivalDeadLine = -1; + return null; } else { Frame frame = queue.remove(0); - if (queue.isEmpty()) { - arrivalDeadLine = -1; - } else { - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); - } + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); //convert duration to nanoseconds frame.setDuration(frame.getDuration() * 1000000L); From 6d280177833302de7cbb81131a79a988ed85dd28 Mon Sep 17 00:00:00 2001 From: Adam Chlupacek Date: Mon, 3 Mar 2025 16:35:59 +0100 Subject: [PATCH 31/33] Fix bugger getting stuck after long silences --- .../media/server/impl/rtp/JitterBuffer.java | 11 +++++- .../server/impl/rtp/JitterBufferTest.java | 38 +++++++++++++++++-- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 163c3b5bc..667b3f615 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -339,7 +339,16 @@ else if (size < BUFFER_SIZE_NOR) { if (dftSize == 0) dftSize = 1; if (frameRateDiff == 0) avgFrameRate = 50; - else avgFrameRate = (int) (dftSize * 1000 / frameRateDiff); + else avgFrameRate = (int) (dftSize * 1000 / frameRateDiff); + + // AVG framerate or last framerate is 0, this would result in a division by zero and the jitter would get stuck + // This case can only happen after long times of inactivity, so we reset the values + if (avgFrameRate == 0 || lastFrameRate == 0) { + avgFrameRate = 50; + lastFrameRate = 50; + decodedFrameTime.clear(); + queue.clear(); + } if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) { diff --git a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java index 4167aacd2..85c25cfc7 100644 --- a/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java +++ b/io/rtp/src/test/java/org/mobicents/media/server/impl/rtp/JitterBufferTest.java @@ -23,13 +23,11 @@ package org.mobicents.media.server.impl.rtp; import java.net.InetSocketAddress; +import java.util.Arrays; import java.util.LinkedList; import java.util.Random; import java.util.HashMap; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -39,6 +37,8 @@ import org.mobicents.media.server.scheduler.PriorityQueueScheduler; import org.mobicents.media.server.spi.memory.Frame; +import static org.junit.Assert.*; + /** * * kulikov @@ -74,6 +74,38 @@ public void setUp() { public void tearDown() { } + + @Test + public void testNoPacketsAfter3Packets() throws Exception { + RtpPacket[] stream = createStream(100); + + Frame[] media = new Frame[stream.length]; + for (int i = 0; i < stream.length; i++) { + if (i == 3) { + // Wait for 3020ms while reading and not writing any packets. + for (int j = 0; j < 50 * 3; j++) { + wallClock.tick(20000000L); + media[i] = jitterBuffer.read(wallClock.getTime()); + } + wallClock.tick(20000000L); + } else { + wallClock.tick(20000000L); + } + jitterBuffer.write(stream[i], AVProfile.audio.find(8)); + media[i] = jitterBuffer.read(wallClock.getTime()); + } + + + for (int i = 0; i < media.length; i++) { + Frame f = media[i]; + if (i == 0 || i == 1 || i == 4 || i == 5) { + assertNull("Frames should be missing", f); + } else { + assertNotNull("Frames should be present", f); + } + } + } + @Test public void failingOutOforder() throws Exception { RtpPacket[] stream = createStream(100); From 3a121c8fe5fce473fc63c04cd2433b75746b60db Mon Sep 17 00:00:00 2001 From: Milan Date: Tue, 4 Mar 2025 08:59:36 +0100 Subject: [PATCH 32/33] Update io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java --- .../java/org/mobicents/media/server/impl/rtp/JitterBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index 667b3f615..c007156ef 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -339,7 +339,7 @@ else if (size < BUFFER_SIZE_NOR) { if (dftSize == 0) dftSize = 1; if (frameRateDiff == 0) avgFrameRate = 50; - else avgFrameRate = (int) (dftSize * 1000 / frameRateDiff); + else avgFrameRate = (int) (dftSize * 1000 / frameRateDiff); // AVG framerate or last framerate is 0, this would result in a division by zero and the jitter would get stuck // This case can only happen after long times of inactivity, so we reset the values From 30c3aef19ee5031699411ed4aa19e9ae8209a96f Mon Sep 17 00:00:00 2001 From: Milan Raulim Date: Tue, 4 Mar 2025 09:19:15 +0100 Subject: [PATCH 33/33] Fix arrivalDeadline --- .../media/server/impl/rtp/JitterBuffer.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java index c007156ef..0fa2f9a53 100644 --- a/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java +++ b/io/rtp/src/main/java/org/mobicents/media/server/impl/rtp/JitterBuffer.java @@ -320,13 +320,6 @@ else if (size < BUFFER_SIZE_NOR) { } } - if (queue.isEmpty()) { - arrivalDeadLine = -1; - } else { - //set arrival deadline for the next frame (in rtp time - arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); - } - //convert duration to nanoseconds frame.setDuration(frame.getDuration() * 1000000L); frame.setTimestamp(frame.getTimestamp() * 1000000L); @@ -350,6 +343,12 @@ else if (size < BUFFER_SIZE_NOR) { queue.clear(); } + if (queue.isEmpty()) { + arrivalDeadLine = -1; + } else { + //set arrival deadline for the next frame (in rtp time + arrivalDeadLine = rtpClock.convertToRtpTime(frame.getTimestamp() + frame.getDuration()); + } if (decodedFrameTime.size() >= NUM_FRAME_TIME_HISTORY) { decodedFrameTime.removeLast();