From b75da7c41ea49c78ac9362f8837d4df9469613c6 Mon Sep 17 00:00:00 2001 From: Vito Caputo Date: Tue, 14 Jul 2020 15:46:16 -0700 Subject: timer: implement AV-sync This is focused on keeping --on-the-fly-encoding in sync even over long videos. The existing code inevitably would fall into a permanently negative pdata->avd value letting things get increasingly out of sync and never correcting. Before removing the vestigial negative avd "don't wait" logic from get_frame when this permanently negative avd state was entered, get_frame would just start sampling at an unregulated fps. The timer thread which drives get_frame now consults avd on every tick, Depending on which which half is ahead, the timer will either cause get_frame to drop frames by advancing the frameno by more than one, or it will adjust its sleep delay in proportion to the delta. See comments in rmd_timer.c for more details. Note that in testing especially with a loaded system I observed some surprisingly large deltas where multi-second sleeps occurred to let the sound catch back up. I expect to revisit this issue more in the future, but would just like to get things more correct for now. --- src/rmd_encode_image_buffer.c | 5 +++- src/rmd_encode_sound_buffer.c | 3 +- src/rmd_timer.c | 65 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/rmd_encode_image_buffer.c b/src/rmd_encode_image_buffer.c index 57ac5ce..d83c694 100644 --- a/src/rmd_encode_image_buffer.c +++ b/src/rmd_encode_image_buffer.c @@ -74,10 +74,13 @@ void *rmdEncodeImageBuffer(ProgData *pdata) { while (theora_encode_packetout(&enc_data->m_th_st, 0, &enc_data->m_ogg_pckt1) > 0) { pthread_mutex_lock(&pdata->libogg_mutex); ogg_stream_packetin(&enc_data->m_ogg_ts, &enc_data->m_ogg_pckt1); - pdata->avd += pdata->frametime; pthread_mutex_unlock(&pdata->libogg_mutex); } + pthread_mutex_lock(&pdata->avd_mutex); + pdata->avd += pdata->frametime * n_frames; + pthread_mutex_unlock(&pdata->avd_mutex); + last_encode_frameno = encode_frameno; } diff --git a/src/rmd_encode_sound_buffer.c b/src/rmd_encode_sound_buffer.c index d35a17c..22e4ddc 100644 --- a/src/rmd_encode_sound_buffer.c +++ b/src/rmd_encode_sound_buffer.c @@ -128,8 +128,9 @@ void *rmdEncodeSoundBuffer(ProgData *pdata) { } pthread_mutex_unlock(&pdata->libogg_mutex); - /* this needs synchronizing */ + pthread_mutex_lock(&pdata->avd_mutex); pdata->avd -= pdata->periodtime; + pthread_mutex_unlock(&pdata->avd_mutex); } pthread_mutex_lock(&pdata->vorbis_lib_mutex); diff --git a/src/rmd_timer.c b/src/rmd_timer.c index ff9d068..be757a6 100644 --- a/src/rmd_timer.c +++ b/src/rmd_timer.c @@ -39,15 +39,65 @@ #include -void *rmdTimer(ProgData *pdata){ - struct timespec delay; +static struct timespec us_to_timespec(unsigned int us) +{ + return (struct timespec){ + .tv_sec = us / 1000000, + .tv_nsec = (us % 1000000) * 1000, + }; +} - rmdThreadsSetName("rmdTimer"); +static void sync_streams(ProgData *pdata, unsigned int *frame_step, struct timespec *delay) { + int avd; + + pthread_mutex_lock(&pdata->avd_mutex); + avd = pdata->avd; + pthread_mutex_unlock(&pdata->avd_mutex); + + /* There are two knobs available for keeping the video synchronized with the audio: + * 1. frame_step; how many frames to encode from this frame (aka dropping frames if > 1) + * 2. delay; how long to delay the next get_frame + * + * When avd is negative, we need more video relative to audio. That can be achieved + * by either sleeping less between frames, or dropping them by having the encoder + * encode a given frame multiple times. + * + * When avd is positive, we need more audio relative to video, so less video. This + * can be achieved by sleeping more between frames. + */ + + if (avd < 0) { + int frames_behind = -avd / pdata->frametime; + + if (frames_behind > 0) { + /* more than a whole frame behind, drop frames to catch up */ + *frame_step += frames_behind; + } else { + /* less than a whole frame behind, just sleep less */ + *delay = us_to_timespec(pdata->frametime + avd); + } - delay.tv_sec = 1.f / pdata->args.fps; - delay.tv_nsec = 1000000000.f / pdata->args.fps - delay.tv_sec * 1000000000.f; + } else if (avd > 0) { + /* sleep longer */ + *delay = us_to_timespec(pdata->frametime + avd); + } + +#if 0 + printf("avd: %i frame_step: %u delay: %lu,%lu\n", + avd, *frame_step, (*delay).tv_sec, (*delay).tv_nsec); +#endif +} + +void *rmdTimer(ProgData *pdata) { + + rmdThreadsSetName("rmdTimer"); while (pdata->timer_alive) { + struct timespec delay; + unsigned int frame_step = 1; + + delay.tv_sec = 1.f / pdata->args.fps; + delay.tv_nsec = 1000000000.f / pdata->args.fps - delay.tv_sec * 1000000000.f; pthread_mutex_lock(&pdata->pause_mutex); if (pdata->pause_state_changed) { @@ -71,8 +121,11 @@ void *rmdTimer(ProgData *pdata){ } else pthread_mutex_unlock(&pdata->pause_mutex); + if (!pdata->args.nosound) + sync_streams(pdata, &frame_step, &delay); + pthread_mutex_lock(&pdata->time_mutex); - pdata->time_frameno++; + pdata->time_frameno += frame_step; pthread_mutex_unlock(&pdata->time_mutex); pthread_cond_signal(&pdata->time_cond); -- cgit v1.2.1