timer: implement AV-sync

This is focused on keeping --on-the-fly-encoding in sync even
over long videos.  The existing code inevitably would fall into a
permanently negative pdata->avd value letting things get
increasingly out of sync and never correcting.

Before removing the vestigial negative avd "don't wait" logic
from get_frame when this permanently negative avd state was
entered, get_frame would just start sampling at an unregulated
fps.

The timer thread which drives get_frame now consults avd on every
tick, Depending on which which half is ahead, the timer will
either cause get_frame to drop frames by advancing the frameno by
more than one, or it will adjust its sleep delay in proportion to
the delta.

See comments in rmd_timer.c for more details.

Note that in testing especially with a loaded system I observed
some surprisingly large deltas where multi-second sleeps occurred
to let the sound catch back up.  I expect to revisit this issue
more in the future, but would just like to get things more
correct for now.

3 files changed, 65 insertions, 8 deletions

diff --git a/src/rmd_encode_image_buffer.c b/src/rmd_encode_image_buffer.c
index 57ac5ce..d83c694 100644
--- a/src/rmd_encode_image_buffer.c
+++ b/src/rmd_encode_image_buffer.c
@@ -74,10 +74,13 @@ void *rmdEncodeImageBuffer(ProgData *pdata) {
 		while (theora_encode_packetout(&enc_data->m_th_st, 0, &enc_data->m_ogg_pckt1) > 0) {
 			pthread_mutex_lock(&pdata->libogg_mutex);
 			ogg_stream_packetin(&enc_data->m_ogg_ts, &enc_data->m_ogg_pckt1);
-			pdata->avd += pdata->frametime;
 			pthread_mutex_unlock(&pdata->libogg_mutex);
 		}
 
+		pthread_mutex_lock(&pdata->avd_mutex);
+		pdata->avd += pdata->frametime * n_frames;
+		pthread_mutex_unlock(&pdata->avd_mutex);
+
 		last_encode_frameno = encode_frameno;
 	}
 
diff --git a/src/rmd_encode_sound_buffer.c b/src/rmd_encode_sound_buffer.c
index d35a17c..22e4ddc 100644
--- a/src/rmd_encode_sound_buffer.c
+++ b/src/rmd_encode_sound_buffer.c
@@ -128,8 +128,9 @@ void *rmdEncodeSoundBuffer(ProgData *pdata) {
 		}
 		pthread_mutex_unlock(&pdata->libogg_mutex);
 
-		/* this needs synchronizing */
+		pthread_mutex_lock(&pdata->avd_mutex);
 		pdata->avd -= pdata->periodtime;
+		pthread_mutex_unlock(&pdata->avd_mutex);
 	}
 
 	pthread_mutex_lock(&pdata->vorbis_lib_mutex);
diff --git a/src/rmd_timer.c b/src/rmd_timer.c
index ff9d068..be757a6 100644
--- a/src/rmd_timer.c
+++ b/src/rmd_timer.c
@@ -39,15 +39,65 @@
 #include <unistd.h>
 
 
-void *rmdTimer(ProgData *pdata){
-	struct timespec	delay;
+static struct timespec us_to_timespec(unsigned int us)
+{
+	return (struct timespec){
+		.tv_sec = us / 1000000,
+		.tv_nsec = (us % 1000000) * 1000,
+	};
+}
 
-	rmdThreadsSetName("rmdTimer");
+static void sync_streams(ProgData *pdata, unsigned int *frame_step, struct timespec *delay) {
+	int	avd;
+
+	pthread_mutex_lock(&pdata->avd_mutex);
+	avd = pdata->avd;
+	pthread_mutex_unlock(&pdata->avd_mutex);
+
+	/* There are two knobs available for keeping the video synchronized with the audio:
+	 * 1. frame_step; how many frames to encode from this frame (aka dropping frames if > 1)
+	 * 2. delay; how long to delay the next get_frame
+	 *
+	 * When avd is negative, we need more video relative to audio.  That can be achieved
+	 * by either sleeping less between frames, or dropping them by having the encoder
+	 * encode a given frame multiple times.
+	 *
+	 * When avd is positive, we need more audio relative to video, so less video.  This
+	 * can be achieved by sleeping more between frames.
+	 */
+
+	if (avd < 0) {
+		int	frames_behind = -avd / pdata->frametime;
+
+		if (frames_behind > 0) {
+			/* more than a whole frame behind, drop frames to catch up */
+			*frame_step += frames_behind;
+		} else {
+			/* less than a whole frame behind, just sleep less */
+			*delay = us_to_timespec(pdata->frametime + avd);
+		}
 
-	delay.tv_sec = 1.f / pdata->args.fps;
-	delay.tv_nsec = 1000000000.f / pdata->args.fps - delay.tv_sec * 1000000000.f;
+	} else if (avd > 0) {
+		/* sleep longer */
+		*delay = us_to_timespec(pdata->frametime + avd);
+	}
+
+#if 0
+	printf("avd: %i frame_step: %u delay: %lu,%lu\n",
+		avd, *frame_step, (*delay).tv_sec, (*delay).tv_nsec);
+#endif
+}
+
+void *rmdTimer(ProgData *pdata) {
+
+	rmdThreadsSetName("rmdTimer");
 
 	while (pdata->timer_alive) {
+		struct timespec	delay;
+		unsigned int	frame_step = 1;
+
+		delay.tv_sec = 1.f / pdata->args.fps;
+		delay.tv_nsec = 1000000000.f / pdata->args.fps - delay.tv_sec * 1000000000.f;
 
 		pthread_mutex_lock(&pdata->pause_mutex);
 		if (pdata->pause_state_changed) {
@@ -71,8 +121,11 @@ void *rmdTimer(ProgData *pdata){
 		} else
 			pthread_mutex_unlock(&pdata->pause_mutex);
 
+		if (!pdata->args.nosound)
+			sync_streams(pdata, &frame_step, &delay);
+
 		pthread_mutex_lock(&pdata->time_mutex);
-		pdata->time_frameno++;
+		pdata->time_frameno += frame_step;
 		pthread_mutex_unlock(&pdata->time_mutex);
 		pthread_cond_signal(&pdata->time_cond);