diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h index 25f83c7cf35..aacbaab2de8 100644 --- a/api/video/encoded_image.h +++ b/api/video/encoded_image.h @@ -212,6 +212,14 @@ class RTC_EXPORT EncodedImage { int64_t receive_finish_ms = 0; } timing_; +#if defined(WEBRTC_WIN) + struct BWEStats { + double start_duration_ = 0; + double last_duration_ = 0; + int32_t packets_lost_ = 0; + }bwe_stats_; +#endif + private: // TODO(bugs.webrtc.org/9378): We're transitioning to always owning the // encoded data. diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index f7988763d38..b077f379fa7 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -48,7 +48,15 @@ class RtpFrameObject : public EncodedFrame { bool delayed_by_retransmission() const override; const RTPVideoHeader& GetRtpVideoHeader() const; const FrameMarking& GetFrameMarking() const; - +#if defined(WEBRTC_WIN) + void SetBWETiming(double start_duration, + double last_duration, + int32_t packets_lost) { + bwe_stats_.start_duration_ = start_duration; + bwe_stats_.last_duration_ = last_duration; + bwe_stats_.packets_lost_ = packets_lost; + } +#endif private: RTPVideoHeader rtp_video_header_; VideoCodecType codec_type_; diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc index 15c8f83e146..60ce8d1d66e 100644 --- a/modules/video_coding/packet_buffer.cc +++ b/modules/video_coding/packet_buffer.cc @@ -102,6 +102,10 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket( first_seq_num_ = seq_num; } +#if defined(WEBRTC_WIN) + QueryPerformanceCounter((LARGE_INTEGER*)&packet->time_ticks); +#endif + if (buffer_[index] != nullptr) { // Duplicate packet, just delete the payload. if (buffer_[index]->seq_num == packet->seq_num) { diff --git a/modules/video_coding/packet_buffer.h b/modules/video_coding/packet_buffer.h index c480e372395..9de80510720 100644 --- a/modules/video_coding/packet_buffer.h +++ b/modules/video_coding/packet_buffer.h @@ -70,6 +70,9 @@ class PacketBuffer { RTPVideoHeader video_header; RtpPacketInfo packet_info; +#if defined(WEBRTC_WIN) + int64_t time_ticks; +#endif }; struct InsertResult { std::vector> packets; diff --git a/rtc_base/time/BUILD.gn b/rtc_base/time/BUILD.gn index e13ccd35eed..2e2e3dfedd1 100644 --- a/rtc_base/time/BUILD.gn +++ b/rtc_base/time/BUILD.gn @@ -17,5 +17,8 @@ rtc_library("timestamp_extrapolator") { "timestamp_extrapolator.cc", "timestamp_extrapolator.h", ] + if(is_win) { + sources += [ "ptp_clock_sync.h" ] + } deps = [ "../synchronization:rw_lock_wrapper" ] } diff --git a/rtc_base/time/ptp_clock_sync.h b/rtc_base/time/ptp_clock_sync.h new file mode 100644 index 00000000000..a4ad2631da5 --- /dev/null +++ b/rtc_base/time/ptp_clock_sync.h @@ -0,0 +1,57 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef RTC_BASE_TIME_PTP_CLOCK_SYNC_H_ +#define RTC_BASE_TIME_PTP_CLOCK_SYNC_H_ + +#include +#include + +#define MICROSECONDS_FACTOR 1000000.0 +#define OFFSET_FACTOR 200000 +#define SERVER_FREQUENCY 0.09 // RTP/NTP timestamp runs at 90KHz clock + +// A Windows implementation for PTP using timestamp from RTP and local +// timestamp. We may need to implement something like QueryPerformanceFrequency +// for this to work on Linux platforms. +namespace webrtc { +class PTPClockSync { + public: + PTPClockSync() + : m_server_point(0), m_server_freq(0), m_client_point(0), m_last_ts(0) { + uint64_t freq; // Performance counter frequency in a second. + QueryPerformanceFrequency((LARGE_INTEGER*)&freq); + m_client_freq = (double)freq / MICROSECONDS_FACTOR; + m_server_freq = SERVER_FREQUENCY; + } + ~PTPClockSync() {} + + void Sync(uint32_t ts, uint64_t tc) { + if (GetDuration(ts, tc) < 0 || + (ts - m_last_ts) > OFFSET_FACTOR * m_server_freq) { + UpdateSync(ts, tc); + } + m_last_ts = ts; + } + + double GetDuration(uint32_t ts, uint64_t tc) { + int ds = (int)(ts - m_server_point); + int dc = (int)(tc - m_client_point); + return (double)dc / m_client_freq - (double)ds / m_server_freq; + } + + protected: + uint32_t m_server_point; + double m_server_freq; // count per us + uint64_t m_client_point; + double m_client_freq; // count per us + uint32_t m_last_ts; + + void UpdateSync(uint32_t ts, uint64_t tc) { + m_client_point = tc; + m_server_point = ts; + } +}; +} // namespace webrtc +#endif \ No newline at end of file diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index f2eb64acf3c..a8e309b82fd 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -523,8 +523,11 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( RTC_LOG(LS_INFO) << "LossNotificationController does not support reordering."; } else if (generic_descriptor_state == kNoGenericDescriptor) { +#if 0 + // Johny(TODO): check cause of no generic descriptor. RTC_LOG(LS_WARNING) << "LossNotificationController requires generic " "frame descriptor, but it is missing."; +#endif } else { if (video_header.is_first_packet_in_frame) { RTC_DCHECK(video_header.generic); @@ -742,6 +745,11 @@ void RtpVideoStreamReceiver::OnInsertedPacket( std::vector> payloads; RtpPacketInfos::vector_type packet_infos; + // Add timing information required by sender-side BWE. +#if defined(WEBRTC_WIN) + int64_t max_tc = 0, min_tc = 0; + double start_duration = 0, last_duration = 0; +#endif bool frame_boundary = true; for (auto& packet : result.packets) { // PacketBuffer promisses frame boundaries are correctly set on each @@ -754,18 +762,30 @@ void RtpVideoStreamReceiver::OnInsertedPacket( max_recv_time = packet->packet_info.receive_time_ms(); payloads.clear(); packet_infos.clear(); +#if defined(WEBRTC_WIN) + max_tc = min_tc = packet->time_ticks; +#endif } else { max_nack_count = std::max(max_nack_count, packet->times_nacked); min_recv_time = std::min(min_recv_time, packet->packet_info.receive_time_ms()); max_recv_time = std::max(max_recv_time, packet->packet_info.receive_time_ms()); +#if defined(WEBRTC_WIN) + max_tc = std::max(max_tc, packet->time_ticks); + min_tc = std::min(min_tc, packet->time_ticks); +#endif } payloads.emplace_back(packet->video_payload); packet_infos.push_back(packet->packet_info); frame_boundary = packet->is_last_packet_in_frame(); if (packet->is_last_packet_in_frame()) { +#if defined(WEBRTC_WIN) + clock_sync_.Sync(packet->timestamp, min_tc); + start_duration = clock_sync_.GetDuration(packet->timestamp, min_tc); + last_duration = clock_sync_.GetDuration(packet->timestamp, max_tc); +#endif auto depacketizer_it = payload_type_map_.find(first_packet->payload_type); RTC_CHECK(depacketizer_it != payload_type_map_.end()); @@ -777,24 +797,35 @@ void RtpVideoStreamReceiver::OnInsertedPacket( } const video_coding::PacketBuffer::Packet& last_packet = *packet; - OnAssembledFrame(std::make_unique( - first_packet->seq_num, // - last_packet.seq_num, // - last_packet.marker_bit, // - max_nack_count, // - min_recv_time, // - max_recv_time, // - first_packet->timestamp, // - first_packet->ntp_time_ms, // - last_packet.video_header.video_timing, // - first_packet->payload_type, // - first_packet->codec(), // - last_packet.video_header.rotation, // - last_packet.video_header.content_type, // - first_packet->video_header, // - last_packet.video_header.color_space, // - RtpPacketInfos(std::move(packet_infos)), // - std::move(bitstream))); + std::unique_ptr frame = + std::make_unique( + first_packet->seq_num, // + last_packet.seq_num, // + last_packet.marker_bit, // + max_nack_count, // + min_recv_time, // + max_recv_time, // + first_packet->timestamp, // + first_packet->ntp_time_ms, // + last_packet.video_header.video_timing, // + first_packet->payload_type, // + first_packet->codec(), // + last_packet.video_header.rotation, // + last_packet.video_header.content_type, // + first_packet->video_header, // + last_packet.video_header.color_space, // + RtpPacketInfos(std::move(packet_infos)), // + std::move(bitstream)); +#if defined(WEBRTC_WIN) + StreamStatistician* ss = + rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc); + int32_t packets_lost = 0; + if (ss != nullptr) { + packets_lost = ss->GetStats().packets_lost; + frame->SetBWETiming(start_duration, last_duration, packets_lost); + } +#endif + OnAssembledFrame(std::move(frame)); } } RTC_DCHECK(frame_boundary); diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index f4dc06dbec0..0b7f9e7a47f 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -47,6 +47,9 @@ #include "rtc_base/constructor_magic.h" #include "rtc_base/critical_section.h" #include "rtc_base/numerics/sequence_number_util.h" +#if defined(WEBRTC_WIN) +#include "rtc_base/time/ptp_clock_sync.h" +#endif #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/thread_annotations.h" #include "rtc_base/thread_checker.h" @@ -272,6 +275,9 @@ class RtpVideoStreamReceiver : public LossNotificationSender, void OnAssembledFrame(std::unique_ptr frame); Clock* const clock_; +#if defined(WEBRTC_WIN) + PTPClockSync clock_sync_; +#endif // Ownership of this object lies with VideoReceiveStream, which owns |this|. const VideoReceiveStream::Config& config_; PacketRouter* const packet_router_;