feat: audio source timestamps

livekit-ffi/src/server/requests.rs

@@ -444,6 +444,7 @@ fn remix_and_resample(
         num_channels: remix.num_channels,
         samples_per_channel: data_len,
         sample_rate: remix.sample_rate,
+        timestamp_ms: 0,
     };
 
     let handle_id = server.next_id();

livekit-webrtc/src/audio_frame.rs

@@ -18,6 +18,7 @@ pub struct AudioFrame {
     pub sample_rate: u32,
     pub num_channels: u32,
     pub samples_per_channel: u32,
+    pub timestamp_ms: i64,
 }
 
 impl AudioFrame {
@@ -27,6 +28,7 @@ impl AudioFrame {
             sample_rate,
             num_channels,
             samples_per_channel,
+            timestamp_ms: 0,
         }
     }
 }

livekit-webrtc/src/native/audio_source.rs

@@ -15,7 +15,10 @@
 use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions};
 use cxx::SharedPtr;
 use parking_lot::Mutex;
-use std::sync::Arc;
+use std::{
+    sync::Arc,
+    time::{SystemTime, UNIX_EPOCH},
+};
 use webrtc_sys::audio_track as sys_at;
 
 impl From<sys_at::ffi::AudioSourceOptions> for AudioSourceOptions {
@@ -83,6 +86,12 @@ impl NativeAudioSource {
             inner.num_channels = frame.num_channels;
         }
 
+        let mut timestamp = frame.timestamp_ms;
+        if timestamp == 0 {
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+            timestamp = now.as_millis() as i64;
+        }
+
         // Split the frame into 10ms chunks
         let mut i = 0;
         loop {
@@ -116,8 +125,10 @@ impl NativeAudioSource {
                 frame.sample_rate as i32,
                 frame.num_channels as usize,
                 samples_10ms / frame.num_channels as usize,
+                timestamp,
             );
 
+            timestamp += 10;
             i += needed_data;
         }
     }

livekit-webrtc/src/native/audio_stream.rs

@@ -84,6 +84,7 @@ impl sys_at::AudioSink for AudioTrackObserver {
             sample_rate: sample_rate as u32,
             num_channels: nb_channels as u32,
             samples_per_channel: nb_frames as u32,
+            timestamp_ms: 0, // TODO(theomonnom): How can we receive timestamp here?
         });
     }
 }

webrtc-sys/include/livekit/audio_track.h

@@ -16,6 +16,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 
 #include "api/audio_options.h"
@@ -24,6 +25,7 @@
 #include "livekit/webrtc.h"
 #include "pc/local_audio_source.h"
 #include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/timestamp_aligner.h"
 #include "rust/cxx.h"
 
 namespace livekit {
@@ -96,10 +98,12 @@ class AudioTrackSource {
     void on_captured_frame(rust::Slice<const int16_t> audio_data,
                            int sample_rate,
                            size_t number_of_channels,
-                           size_t number_of_frames);
+                           size_t number_of_frames,
+                           int64_t timestamp_ms);
 
    private:
     mutable webrtc::Mutex mutex_;
+    rtc::TimestampAligner timestamp_aligner_;
     std::vector<webrtc::AudioTrackSinkInterface*> sinks_;
     cricket::AudioOptions options_{};
   };
@@ -114,7 +118,8 @@ class AudioTrackSource {
   void on_captured_frame(rust::Slice<const int16_t> audio_data,
                          int sample_rate,
                          size_t number_of_channels,
-                         size_t number_of_frames) const;
+                         size_t number_of_frames,
+                         int64_t timestamp_ms) const;
 
   rtc::scoped_refptr<InternalSource> get() const;
 

webrtc-sys/src/audio_track.cpp

@@ -17,6 +17,7 @@
 #include "livekit/audio_track.h"
 
 #include <algorithm>
+#include <cstdint>
 #include <iostream>
 #include <memory>
 
@@ -133,11 +134,12 @@ void AudioTrackSource::InternalSource::on_captured_frame(
     rust::Slice<const int16_t> data,
     int sample_rate,
     size_t number_of_channels,
-    size_t number_of_frames) {
+    size_t number_of_frames,
+    int64_t timestamp_ms) {
   webrtc::MutexLock lock(&mutex_);
   for (auto sink : sinks_) {
     sink->OnData(data.data(), 16, sample_rate, number_of_channels,
-                 number_of_frames);
+                 number_of_frames, timestamp_ms);
   }
 }
 
@@ -158,9 +160,10 @@ void AudioTrackSource::set_audio_options(
 void AudioTrackSource::on_captured_frame(rust::Slice<const int16_t> audio_data,
                                          int sample_rate,
                                          size_t number_of_channels,
-                                         size_t number_of_frames) const {
+                                         size_t number_of_frames,
+                                         int64_t timestamp_ms) const {
   source_->on_captured_frame(audio_data, sample_rate, number_of_channels,
-                             number_of_frames);
+                             number_of_frames, timestamp_ms);
 }
 
 rtc::scoped_refptr<AudioTrackSource::InternalSource> AudioTrackSource::get()

webrtc-sys/src/audio_track.rs

@@ -47,6 +47,7 @@ pub mod ffi {
             sample_rate: i32,
             nb_channels: usize,
             nb_frames: usize,
+            timestamp_ms: i64,
         );
         fn audio_options(self: &AudioTrackSource) -> AudioSourceOptions;
         fn set_audio_options(self: &AudioTrackSource, options: &AudioSourceOptions);