diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 3ddd5bd0d..32e94d6b0 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -444,6 +444,7 @@ fn remix_and_resample( num_channels: remix.num_channels, samples_per_channel: data_len, sample_rate: remix.sample_rate, + timestamp_ms: 0, }; let handle_id = server.next_id(); diff --git a/livekit-webrtc/src/audio_frame.rs b/livekit-webrtc/src/audio_frame.rs index a3c537b11..cfcacfbee 100644 --- a/livekit-webrtc/src/audio_frame.rs +++ b/livekit-webrtc/src/audio_frame.rs @@ -18,6 +18,7 @@ pub struct AudioFrame { pub sample_rate: u32, pub num_channels: u32, pub samples_per_channel: u32, + pub timestamp_ms: i64, } impl AudioFrame { @@ -27,6 +28,7 @@ impl AudioFrame { sample_rate, num_channels, samples_per_channel, + timestamp_ms: 0, } } } diff --git a/livekit-webrtc/src/native/audio_source.rs b/livekit-webrtc/src/native/audio_source.rs index 91c6b0351..7c533c01c 100644 --- a/livekit-webrtc/src/native/audio_source.rs +++ b/livekit-webrtc/src/native/audio_source.rs @@ -15,7 +15,10 @@ use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions}; use cxx::SharedPtr; use parking_lot::Mutex; -use std::sync::Arc; +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; use webrtc_sys::audio_track as sys_at; impl From for AudioSourceOptions { @@ -83,6 +86,12 @@ impl NativeAudioSource { inner.num_channels = frame.num_channels; } + let mut timestamp = frame.timestamp_ms; + if timestamp == 0 { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + timestamp = now.as_millis() as i64; + } + // Split the frame into 10ms chunks let mut i = 0; loop { @@ -116,8 +125,10 @@ impl NativeAudioSource { frame.sample_rate as i32, frame.num_channels as usize, samples_10ms / frame.num_channels as usize, + timestamp, ); + timestamp += 10; i += needed_data; } } diff --git a/livekit-webrtc/src/native/audio_stream.rs b/livekit-webrtc/src/native/audio_stream.rs index a3f530907..82f89563a 100644 --- a/livekit-webrtc/src/native/audio_stream.rs +++ b/livekit-webrtc/src/native/audio_stream.rs @@ -84,6 +84,7 @@ impl sys_at::AudioSink for AudioTrackObserver { sample_rate: sample_rate as u32, num_channels: nb_channels as u32, samples_per_channel: nb_frames as u32, + timestamp_ms: 0, // TODO(theomonnom): How can we receive timestamp here? }); } } diff --git a/webrtc-sys/include/livekit/audio_track.h b/webrtc-sys/include/livekit/audio_track.h index 0d08de653..ba919de20 100644 --- a/webrtc-sys/include/livekit/audio_track.h +++ b/webrtc-sys/include/livekit/audio_track.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include "api/audio_options.h" @@ -24,6 +25,7 @@ #include "livekit/webrtc.h" #include "pc/local_audio_source.h" #include "rtc_base/synchronization/mutex.h" +#include "rtc_base/timestamp_aligner.h" #include "rust/cxx.h" namespace livekit { @@ -96,10 +98,12 @@ class AudioTrackSource { void on_captured_frame(rust::Slice audio_data, int sample_rate, size_t number_of_channels, - size_t number_of_frames); + size_t number_of_frames, + int64_t timestamp_ms); private: mutable webrtc::Mutex mutex_; + rtc::TimestampAligner timestamp_aligner_; std::vector sinks_; cricket::AudioOptions options_{}; }; @@ -114,7 +118,8 @@ class AudioTrackSource { void on_captured_frame(rust::Slice audio_data, int sample_rate, size_t number_of_channels, - size_t number_of_frames) const; + size_t number_of_frames, + int64_t timestamp_ms) const; rtc::scoped_refptr get() const; diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp index eff92a483..1921b0d2b 100644 --- a/webrtc-sys/src/audio_track.cpp +++ b/webrtc-sys/src/audio_track.cpp @@ -17,6 +17,7 @@ #include "livekit/audio_track.h" #include +#include #include #include @@ -133,11 +134,12 @@ void AudioTrackSource::InternalSource::on_captured_frame( rust::Slice data, int sample_rate, size_t number_of_channels, - size_t number_of_frames) { + size_t number_of_frames, + int64_t timestamp_ms) { webrtc::MutexLock lock(&mutex_); for (auto sink : sinks_) { sink->OnData(data.data(), 16, sample_rate, number_of_channels, - number_of_frames); + number_of_frames, timestamp_ms); } } @@ -158,9 +160,10 @@ void AudioTrackSource::set_audio_options( void AudioTrackSource::on_captured_frame(rust::Slice audio_data, int sample_rate, size_t number_of_channels, - size_t number_of_frames) const { + size_t number_of_frames, + int64_t timestamp_ms) const { source_->on_captured_frame(audio_data, sample_rate, number_of_channels, - number_of_frames); + number_of_frames, timestamp_ms); } rtc::scoped_refptr AudioTrackSource::get() diff --git a/webrtc-sys/src/audio_track.rs b/webrtc-sys/src/audio_track.rs index dd92dd4b4..7d8a32d5d 100644 --- a/webrtc-sys/src/audio_track.rs +++ b/webrtc-sys/src/audio_track.rs @@ -47,6 +47,7 @@ pub mod ffi { sample_rate: i32, nb_channels: usize, nb_frames: usize, + timestamp_ms: i64, ); fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions; fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions);