From 44d25eeed23e6bb4b6c95acca38b039d5af99d00 Mon Sep 17 00:00:00 2001 From: Vito Caputo Date: Sun, 8 Nov 2020 20:30:03 -0800 Subject: cache: pivot to new CacheFile API This minimally switches all the ad-hoc image cache files handling over to using CacheFile. I left the audio cache alone for now as it seems to not be compressed. It might make sense in the future to switch that over as well, especially if I start adding features to CacheFile like preallocating and async periodic fdatasync. --- src/rmd_cache.c | 16 ++------ src/rmd_cache_frame.c | 108 +++++++++++--------------------------------------- src/rmd_get_frame.c | 6 +++ src/rmd_load_cache.c | 80 +++++++++++-------------------------- src/rmd_types.h | 41 +++++++++---------- 5 files changed, 76 insertions(+), 175 deletions(-) diff --git a/src/rmd_cache.c b/src/rmd_cache.c index 94c0b68..db7682e 100644 --- a/src/rmd_cache.c +++ b/src/rmd_cache.c @@ -357,18 +357,10 @@ void rmdInitCacheData(ProgData *pdata, EncData *enc_data_t, CacheData *cache_dat exit(13); } - if (!pdata->args.zerocompression) { - cache_data_t->ifp = gzopen(cache_data_t->imgdata, "wb0f"); - if (cache_data_t->ifp == NULL) { - fprintf(stderr, "Could not create temporary file %s !!!\n", cache_data_t->imgdata); - exit(13); - } - } else { - cache_data_t->uncifp = fopen(cache_data_t->imgdata, "wb0f"); - if (cache_data_t->uncifp == NULL) { - fprintf(stderr, "Could not create temporary file %s !!!\n", cache_data_t->imgdata); - exit(13); - } + cache_data_t->icf = rmdCacheFileOpen(pdata, cache_data_t->imgdata, RMD_CACHE_FILE_MODE_WRITE); + if (cache_data_t->icf == NULL) { + fprintf(stderr, "Could not create temporary file %s !!!\n", cache_data_t->imgdata); + exit(13); } if (!pdata->args.nosound) { diff --git a/src/rmd_cache_frame.c b/src/rmd_cache_frame.c index 3a4e9b3..fbec467 100644 --- a/src/rmd_cache_frame.c +++ b/src/rmd_cache_frame.c @@ -42,7 +42,6 @@ #define BYTES_PER_MB (1024 * 1024) #define CACHE_OUT_BUFFER_SIZE (4 * 1024) -#define CACHE_FILE_SIZE_LIMIT (500 * 1024 * 1024) static int rmdFlushBlock( @@ -51,8 +50,7 @@ static int rmdFlushBlock( int width, int height, int blockwidth, - gzFile fp, - FILE *ucfp, + CacheFile *icf, int flush) { @@ -64,11 +62,7 @@ static int rmdFlushBlock( static unsigned int out_buffer_bytes = 0; if (out_buffer_bytes + pow(blockwidth, 2) >= CACHE_OUT_BUFFER_SIZE || (flush && out_buffer_bytes)) { - if (ucfp == NULL) - gzwrite(fp, (void *)out_buffer, out_buffer_bytes); - else - fwrite((void *)out_buffer, 1, out_buffer_bytes, ucfp); - + rmdCacheFileWrite(icf, (void *)out_buffer, out_buffer_bytes); /* XXX: errors! */ bytes_written = out_buffer_bytes; out_buffer_bytes = 0; } @@ -92,15 +86,12 @@ static int rmdFlushBlock( void *rmdCacheImageBuffer(ProgData *pdata) { - gzFile fp = NULL; - FILE *ucfp = NULL; int index_entry_size = sizeof(u_int32_t), blocknum_x = pdata->enc_data->yuv.y_width / Y_UNIT_WIDTH, blocknum_y = pdata->enc_data->yuv.y_height / Y_UNIT_WIDTH, firstrun = 1, frameno = 0, - nbytes = 0, - nth_cache = 1; + nbytes = 0; u_int32_t ynum, unum, vnum, y_short_blocks[blocknum_x * blocknum_y], u_short_blocks[blocknum_x * blocknum_y], @@ -108,20 +99,13 @@ void *rmdCacheImageBuffer(ProgData *pdata) unsigned long long int total_bytes = 0; unsigned long long int total_received_bytes = 0; unsigned int capture_frameno = 0; + CacheFile *icf; rmdThreadsSetName("rmdCacheImages"); - if (!pdata->args.zerocompression) { - fp = pdata->cache_data->ifp; - - if (fp == NULL) - exit(13); - } else { - ucfp = pdata->cache_data->uncifp; - - if (ucfp == NULL) - exit(13); - } + icf = pdata->cache_data->icf; + if (!icf) + exit(13); while (pdata->running) { FrameHeader fheader; @@ -175,11 +159,11 @@ void *rmdCacheImageBuffer(ProgData *pdata) } /**WRITE FRAME TO DISK*/ - if (!pdata->args.zerocompression) { + if (icf->gzfp) { if (ynum * 4 + unum + vnum > (blocknum_x * blocknum_y * 6) / 10) - gzsetparams(fp, 1, Z_FILTERED); + gzsetparams(icf->gzfp, 1, Z_FILTERED); else - gzsetparams(fp, 0, Z_FILTERED); + gzsetparams(icf->gzfp, 0, Z_FILTERED); } strncpy(fheader.frame_prefix, "FRAM", 4); @@ -190,30 +174,16 @@ void *rmdCacheImageBuffer(ProgData *pdata) fheader.Unum = unum; fheader.Vnum = vnum; - if (!pdata->args.zerocompression) { - nbytes += gzwrite(fp, (void*)&fheader, sizeof(FrameHeader)); - //flush indexes - if (ynum) - nbytes += gzwrite(fp, (void*)y_short_blocks, ynum * index_entry_size); - - if (unum) - nbytes += gzwrite(fp, (void*)u_short_blocks, unum * index_entry_size); + nbytes += rmdCacheFileWrite(icf, (void*)&fheader, sizeof(FrameHeader)); + //flush indexes + if (ynum) + nbytes += rmdCacheFileWrite(icf, (void*)y_short_blocks, ynum * index_entry_size); - if (vnum) - nbytes += gzwrite(fp, (void*)v_short_blocks, vnum * index_entry_size); - } else { - nbytes += sizeof(FrameHeader)* - fwrite((void*)&fheader, sizeof(FrameHeader), 1, ucfp); - //flush indexes - if (ynum) - nbytes += index_entry_size * fwrite(y_short_blocks, index_entry_size, ynum, ucfp); + if (unum) + nbytes += rmdCacheFileWrite(icf, (void*)u_short_blocks, unum * index_entry_size); - if (unum) - nbytes += index_entry_size * fwrite(u_short_blocks, index_entry_size, unum, ucfp); - - if (vnum) - nbytes += index_entry_size * fwrite(v_short_blocks, index_entry_size, vnum, ucfp); - } + if (vnum) + nbytes += rmdCacheFileWrite(icf, (void*)v_short_blocks, vnum * index_entry_size); //flush the blocks for each buffer if (ynum) { @@ -223,8 +193,7 @@ void *rmdCacheImageBuffer(ProgData *pdata) pdata->enc_data->yuv.y_width, pdata->enc_data->yuv.y_height, Y_UNIT_WIDTH, - fp, - ucfp, + icf, 0); } @@ -235,8 +204,7 @@ void *rmdCacheImageBuffer(ProgData *pdata) pdata->enc_data->yuv.uv_width, pdata->enc_data->yuv.uv_height, UV_UNIT_WIDTH, - fp, - ucfp, + icf, 0); } @@ -247,38 +215,14 @@ void *rmdCacheImageBuffer(ProgData *pdata) pdata->enc_data->yuv.uv_width, pdata->enc_data->yuv.uv_height, UV_UNIT_WIDTH, - fp, - ucfp, + icf, 0); } //release main buffer pthread_mutex_unlock(&pdata->yuv_mutex); - nbytes += rmdFlushBlock(NULL, 0, 0, 0, 0, fp, ucfp, 1); - /**@________________@**/ - - if (nbytes > CACHE_FILE_SIZE_LIMIT) { - if (rmdSwapCacheFilesWrite(pdata->cache_data->imgdata, nth_cache, &fp, &ucfp)) { - fprintf(stderr, "New cache file could not be created.\n" - "Ending recording...\n"); - raise(SIGINT); //if for some reason we cannot make a new file - //we have to stop. If we are out of space, - //which means - //that encoding cannot happen either, - //InitEncoder will cause an abrupt end with an - //error code and the cache will remain intact. - //If we've chosen separate two-stages, - //the program will make a - //clean exit. - //In either case data will be preserved so if - //space is freed the recording - //can be proccessed later. - } - total_bytes += nbytes; - nth_cache++; - nbytes = 0; - } + nbytes += rmdFlushBlock(NULL, 0, 0, 0, 0, icf, 1); } total_bytes += nbytes; @@ -309,13 +253,7 @@ void *rmdCacheImageBuffer(ProgData *pdata) frameno, capture_frameno); - if (!pdata->args.zerocompression) { - gzflush(fp, Z_FINISH); - gzclose(fp); - } else { - fflush(ucfp); - fclose(ucfp); - } + rmdCacheFileClose(pdata->cache_data->icf); pthread_exit(&errno); } diff --git a/src/rmd_get_frame.c b/src/rmd_get_frame.c index 9524333..1a5f4d0 100644 --- a/src/rmd_get_frame.c +++ b/src/rmd_get_frame.c @@ -262,6 +262,12 @@ static void rmdBlocksFromList( RectArea **root, } } +/* This thread just samples the recorded window in response to rmdTimer's + * triggers, updating the yuv buffer hopefully at the desired frame rate. + * Following every update, the time_frameno is propogated to capture_frameno + * and image_buffer_ready is signaled for the cache/encode side to consume the + * yuv buffer. + */ void *rmdGetFrame(ProgData *pdata) { int blocks_w = pdata->enc_data->yuv.y_width / Y_UNIT_WIDTH, diff --git a/src/rmd_load_cache.c b/src/rmd_load_cache.c index b5ac9fb..4ad1741 100644 --- a/src/rmd_load_cache.c +++ b/src/rmd_load_cache.c @@ -85,18 +85,12 @@ static void rmdLoadBlock( } //returns number of bytes -static int rmdReadZF(void * buffer, size_t size, size_t nmemb, FILE *ucfp, gzFile ifp) +static int rmdReadZF(void * buffer, size_t size, size_t nmemb, CacheFile *icf) { - if ((ifp != NULL && ucfp != NULL) || - (ifp == NULL && ucfp == NULL)) - return -1; - else if (ucfp != NULL) { - return size * fread(buffer, size, nmemb, ucfp); - } else - return gzread(ifp, buffer, size * nmemb); + return rmdCacheFileRead(icf, buffer, size * nmemb); } -static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) +static int rmdReadFrame(CachedFrame *frame, CacheFile *icf) { int index_entry_size = sizeof(u_int32_t); @@ -104,8 +98,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->YBlocks, index_entry_size, frame->header->Ynum, - ucfp, - ifp) != index_entry_size * frame->header->Ynum) { + icf) != index_entry_size * frame->header->Ynum) { return -1; } } @@ -114,8 +107,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->UBlocks, index_entry_size, frame->header->Unum, - ucfp, - ifp) != index_entry_size * frame->header->Unum) { + icf) != index_entry_size * frame->header->Unum) { return -1; } } @@ -124,8 +116,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->VBlocks, index_entry_size, frame->header->Vnum, - ucfp, - ifp) != index_entry_size * frame->header->Vnum) { + icf) != index_entry_size * frame->header->Vnum) { return -1; } } @@ -134,8 +125,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->YData, Y_UNIT_BYTES, frame->header->Ynum, - ucfp, - ifp) != Y_UNIT_BYTES * frame->header->Ynum) { + icf) != Y_UNIT_BYTES * frame->header->Ynum) { return -2; } } @@ -144,8 +134,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->UData, UV_UNIT_BYTES, frame->header->Unum, - ucfp, - ifp) != UV_UNIT_BYTES * frame->header->Unum) { + icf) != UV_UNIT_BYTES * frame->header->Unum) { return -2; } } @@ -154,8 +143,7 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) if (rmdReadZF( frame->VData, UV_UNIT_BYTES, frame->header->Vnum, - ucfp, - ifp) != UV_UNIT_BYTES * frame->header->Vnum) { + icf) != UV_UNIT_BYTES * frame->header->Vnum) { return -2; } } @@ -163,25 +151,19 @@ static int rmdReadFrame(CachedFrame *frame, FILE *ucfp, gzFile ifp) return 0; } -static int read_header(ProgData *pdata, gzFile ifp, FILE *ucfp, FrameHeader *fheader) +static int read_header(FrameHeader *fheader, CacheFile *icf) { - if (!pdata->args.zerocompression) { - return gzread(ifp, fheader, sizeof(FrameHeader)) == sizeof(FrameHeader); - } else { - return fread(fheader, sizeof(FrameHeader), 1, ucfp) == 1; - } + return rmdCacheFileRead(icf, fheader, sizeof(FrameHeader)) == sizeof(FrameHeader); } void *rmdLoadCache(ProgData *pdata) { yuv_buffer *yuv = &pdata->enc_data->yuv; - gzFile ifp = NULL; - FILE *ucfp = NULL; + CacheFile *icf; FILE *afp = pdata->cache_data->afp; FrameHeader fheader; CachedFrame frame; - int nth_cache = 1, - audio_end = 0, + int audio_end = 0, thread_exit = 0, //0 success, -1 couldn't find files,1 couldn't remove blocknum_x = pdata->enc_data->yuv.y_width / Y_UNIT_WIDTH, blocknum_y = pdata->enc_data->yuv.y_height / Y_UNIT_WIDTH, @@ -206,18 +188,10 @@ void *rmdLoadCache(ProgData *pdata) frame.VData = malloc(yuv->uv_width * yuv->uv_height); //and the we open our files - if (!pdata->args.zerocompression) { - ifp = gzopen(pdata->cache_data->imgdata, "rb"); - if (ifp == NULL) { - thread_exit = -1; - pthread_exit(&thread_exit); - } - } else { - ucfp = fopen(pdata->cache_data->imgdata, "rb"); - if (ucfp == NULL) { - thread_exit = -1; - pthread_exit(&thread_exit); - } + icf = rmdCacheFileOpen(pdata, pdata->cache_data->imgdata, RMD_CACHE_FILE_MODE_READ); + if (!icf) { + thread_exit = -1; + pthread_exit(&thread_exit); } if (!pdata->args.nosound) { @@ -238,18 +212,18 @@ void *rmdLoadCache(ProgData *pdata) //video load and encoding if (pdata->avd <= 0 || pdata->args.nosound || audio_end) { - if (read_header(pdata, ifp, ucfp, frame.header)) { + if (read_header(frame.header, icf)) { fprintf(stdout, "\r[%d%%] ", ((frame.header->capture_frameno) * 100) / pdata->capture_frameno); + if (icf->chapter) + fprintf(stdout, "\t[Cache File %d]", icf->chapter); fflush(stdout); if ( (frame.header->Ynum > blocknum_x * blocknum_y) || (frame.header->Unum > blocknum_x * blocknum_y) || (frame.header->Vnum > blocknum_x * blocknum_y) || - rmdReadFrame( &frame, - pdata->args.zerocompression ? ucfp : NULL, - pdata->args.zerocompression ? NULL : ifp) < 0) { + rmdReadFrame(&frame, icf) < 0) { raise(SIGINT); continue; @@ -285,15 +259,7 @@ void *rmdLoadCache(ProgData *pdata) last_capture_frameno = fheader.capture_frameno; } else { - if (rmdSwapCacheFilesRead( pdata->cache_data->imgdata, - nth_cache, - &ifp, - &ucfp)) { - raise(SIGINT); - } else { - fprintf(stderr, "\t[Cache File %d]", nth_cache); - nth_cache++; - } + raise(SIGINT); } //audio load and encoding } else { @@ -326,6 +292,8 @@ void *rmdLoadCache(ProgData *pdata) free(sound_data); + rmdCacheFileClose(icf); + if (!pdata->args.nosound) fclose(afp); diff --git a/src/rmd_types.h b/src/rmd_types.h index 3cc1a84..252861c 100644 --- a/src/rmd_types.h +++ b/src/rmd_types.h @@ -203,28 +203,25 @@ typedef struct CacheFile { //this struct will hold a few basic //information, needed for caching the frames. typedef struct _CacheData{ - char *workdir, //The directory were the project - //will be stored, while recording. - //Since this will take a lot of space, the user must be - //able to change the location. - *projname, //This is the name of the folder that - //will hold the project. - //It is rMD-session-%d where %d is the pid - //of the current proccess. - //This way, running two instances - //will not create problems - //and also, a frontend can identify - //leftovers from a possible crash - //and delete them - *specsfile, //workdir+projname+specs.txt - *imgdata, //workdir+projname+img.out.gz - *audiodata; //workdir+projname+audio.pcm - - gzFile ifp; //image data file pointer - FILE *uncifp; //uncompressed image data file pointer - - FILE *afp; //audio data file pointer - + char *workdir, //The directory were the project + //will be stored, while recording. + //Since this will take a lot of space, the user must be + //able to change the location. + *projname, //This is the name of the folder that + //will hold the project. + //It is rMD-session-%d where %d is the pid + //of the current proccess. + //This way, running two instances + //will not create problems + //and also, a frontend can identify + //leftovers from a possible crash + //and delete them + *specsfile, //workdir+projname+specs.txt + *imgdata, //workdir+projname+img.out.gz + *audiodata; //workdir+projname+audio.pcm + + CacheFile *icf; //image data cache file + FILE *afp; //audio data file pointer }CacheData; //sound buffer -- cgit v1.2.1