diff --git a/doc/filters.texi b/doc/filters.texi index 3534c87c..4ffa475a 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -29982,6 +29982,8 @@ window matching the provided regular expressions is captured. This source does NOT hold a stable FPS. It returns frames at whatever rate the compositor provides them, only capped by the @option{max_framerate}. If you need a stable rate, you need to add an fps filter to drop/duplicate frames as needed. +Note that a static source may not deliver any frames at all, which also stalls +the fps filter; see @option{dup_frames}. If the capture source disappears mid-capture (window closed, monitor disconnected), the filter will return EOF. @@ -30056,6 +30058,13 @@ Crop this many pixels from the left side of the captured frames. If set to 1, return frames with premultiplied alpha. Default is 0 (straight alpha). +@item dup_frames +If enabled (the default), the last frame is output again whenever the source +delivered no new frame within one @option{max_framerate} interval, and black +frames are output until the source delivered its first frame. This keeps the +output flowing while the captured content is static. The output rate is still +not constant, add an @code{fps} filter if needed. + @item resize_mode Defines how the captured content is fitted into the output canvas size. Possible values: diff --git a/libavfilter/vsrc_gfxcapture.c b/libavfilter/vsrc_gfxcapture.c index 19d8a8a5..bb62c4e3 100644 --- a/libavfilter/vsrc_gfxcapture.c +++ b/libavfilter/vsrc_gfxcapture.c @@ -57,6 +57,8 @@ static const AVOption gfxcapture_options[] = { { "crop_bottom", "number of pixels to crop from the bottom of the captured area", OFFSET(crop_bottom), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, { "premultiplied", "return premultiplied alpha frames", OFFSET(premult_alpha), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, + { "dup_frames", "repeat the last frame if the source delivers none in time", + OFFSET(dup_frames), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS }, { "resize_mode", "capture source resize behavior", OFFSET(resize_mode), AV_OPT_TYPE_INT, { .i64 = GFX_RESIZE_CROP }, 0, GFX_RESIZE_NB - 1, FLAGS, .unit = "resize_mode" }, { "crop", "crop or add black bars into frame", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_CROP }, 0, 0, FLAGS, .unit = "resize_mode" }, { "scale", "scale source to fit initial size", 0, AV_OPT_TYPE_CONST, { .i64 = GFX_RESIZE_SCALE }, 0, 0, FLAGS, .unit = "resize_mode" }, diff --git a/libavfilter/vsrc_gfxcapture.h b/libavfilter/vsrc_gfxcapture.h index f541f9ba..bace2bea 100644 --- a/libavfilter/vsrc_gfxcapture.h +++ b/libavfilter/vsrc_gfxcapture.h @@ -67,6 +67,7 @@ typedef struct GfxCaptureContext { int resize_mode; int scale_mode; int premult_alpha; + int dup_frames; } GfxCaptureContext; #ifdef __cplusplus diff --git a/libavfilter/vsrc_gfxcapture_winrt.cpp b/libavfilter/vsrc_gfxcapture_winrt.cpp index ddc6a7df..26d73b8b 100644 --- a/libavfilter/vsrc_gfxcapture_winrt.cpp +++ b/libavfilter/vsrc_gfxcapture_winrt.cpp @@ -150,7 +150,6 @@ struct GfxCaptureContextCpp { volatile int wgc_thread_init_res { INT_MAX }; std::recursive_mutex wgc_thread_uninit_mutex; volatile int wgc_thread_res { 0 }; - std::shared_ptr wgc_thread_cb_data; HWND capture_hwnd { nullptr }; HMONITOR capture_hmonitor { nullptr }; @@ -163,8 +162,12 @@ struct GfxCaptureContextCpp { AVHWFramesContext *frames_ctx { nullptr }; AVD3D11VAFramesContext *frames_hwctx { nullptr }; - int64_t first_pts { 0 }; - int64_t last_pts { 0 }; + int64_t first_pts { AV_NOPTS_VALUE }; + int64_t last_pts { AV_NOPTS_VALUE }; + + // dup_frames: last output frame and the QPC time (100 ns units) the next repeat is due + AVFrame *last_frame { nullptr }; + int64_t next_pts { AV_NOPTS_VALUE }; }; template @@ -722,10 +725,7 @@ static int run_on_wgc_thread(AVFilterContext *avctx, F &&cb) bool cancel; int ret; }; - auto cbdata = ctx->wgc_thread_cb_data ? - std::static_pointer_cast(ctx->wgc_thread_cb_data) : - std::make_shared(); - ctx->wgc_thread_cb_data = cbdata; + auto cbdata = std::make_shared(); cbdata->done = cbdata->cancel = false; cbdata->ret = AVERROR_BUG; @@ -978,6 +978,7 @@ static av_cold void gfxcapture_uninit(AVFilterContext *avctx) stop_wgc_thread(avctx); + av_frame_free(&ctx->last_frame); ctx->d3d.reset(); av_buffer_unref(&ctx->frames_ref); @@ -1268,6 +1269,46 @@ static int gfxcapture_config_props(AVFilterLink *outlink) return 0; } +static HRESULT create_frame_rtv(ID3D11Device *dev, const AVFrame *frame, + const D3D11_TEXTURE2D_DESC &dst_tex_desc, + ID3D11RenderTargetView **rtv) +{ + D3D11_RENDER_TARGET_VIEW_DESC target_desc = {}; + target_desc.Format = dst_tex_desc.Format; + + if (dst_tex_desc.ArraySize > 1) { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2DARRAY; + target_desc.Texture2DArray.ArraySize = 1; + target_desc.Texture2DArray.FirstArraySlice = (uintptr_t)frame->data[1]; + target_desc.Texture2DArray.MipSlice = 0; + } else { + target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D; + target_desc.Texture2D.MipSlice = 0; + } + + return dev->CreateRenderTargetView( + reinterpret_cast(frame->data[0]), &target_desc, rtv); +} + +static int clear_frame(AVFilterContext *avctx, AVFrame *frame) +{ + GfxCaptureContextCpp *ctx = CCTX(avctx->priv)->ctx; + ID3D11DeviceContext *dev_ctx = ctx->device_hwctx->device_context; + + D3D11_TEXTURE2D_DESC dst_tex_desc; + reinterpret_cast(frame->data[0])->GetDesc(&dst_tex_desc); + + ComPtr rtv; + CHECK_HR_RET(create_frame_rtv(ctx->device_hwctx->device, frame, dst_tex_desc, &rtv)); + + const float clear_color[4] = {0.f, 0.f, 0.f, 1.f}; + ctx->device_hwctx->lock(ctx->device_hwctx->lock_ctx); + dev_ctx->ClearRenderTargetView(rtv.Get(), clear_color); + ctx->device_hwctx->unlock(ctx->device_hwctx->lock_ctx); + + return 0; +} + static int render_capture_to_frame(AVFilterContext *avctx, AVFrame *frame, const ComPtr &src_tex) { GfxCaptureContext *cctx = CCTX(avctx->priv); @@ -1285,22 +1326,8 @@ static int render_capture_to_frame(AVFilterContext *avctx, AVFrame *frame, const D3D11_TEXTURE2D_DESC src_tex_desc; src_tex->GetDesc(&src_tex_desc); - D3D11_RENDER_TARGET_VIEW_DESC target_desc = {}; - target_desc.Format = dst_tex_desc.Format; - - if (dst_tex_desc.ArraySize > 1) { - target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2DARRAY; - target_desc.Texture2DArray.ArraySize = 1; - target_desc.Texture2DArray.FirstArraySlice = (uintptr_t)frame->data[1]; - target_desc.Texture2DArray.MipSlice = 0; - } else { - target_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D; - target_desc.Texture2D.MipSlice = 0; - } - ComPtr rtv; - CHECK_HR_RET(dev->CreateRenderTargetView( - reinterpret_cast(frame->data[0]), &target_desc, &rtv)); + CHECK_HR_RET(create_frame_rtv(dev, frame, dst_tex_desc, &rtv)); ComPtr srv; CHECK_HR_RET(dev->CreateShaderResourceView(src_tex.Get(), nullptr, &srv)); @@ -1389,11 +1416,73 @@ static int render_capture_to_frame(AVFilterContext *avctx, AVFrame *frame, const ComPtr cmd_list; CHECK_HR_RET(def_ctx->FinishCommandList(FALSE, &cmd_list)); + ctx->device_hwctx->lock(ctx->device_hwctx->lock_ctx); dev_ctx->ExecuteCommandList(cmd_list.Get(), FALSE); + ctx->device_hwctx->unlock(ctx->device_hwctx->lock_ctx); return 0; } +static void set_frame_props(GfxCaptureContextCpp *ctx, AVFrame *frame) +{ + frame->sample_aspect_ratio = AVRational{1, 1}; + + if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { + // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_LINEAR; + frame->colorspace = AVCOL_SPC_RGB; + } else { + // According to MSDN, all integer formats contain sRGB image data + frame->color_range = AVCOL_RANGE_JPEG; + frame->color_primaries = AVCOL_PRI_BT709; + frame->color_trc = AVCOL_TRC_IEC61966_2_1; + frame->colorspace = AVCOL_SPC_RGB; + } +} + +// Current time in the clock and unit of WGC's SystemRelativeTime (QPC, 100 ns) +static int64_t qpc_now() +{ + static const int64_t freq = []() { + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + return f.QuadPart; + }(); + + LARGE_INTEGER now; + QueryPerformanceCounter(&now); + + if (freq == TIMESPAN_RES64) + return now.QuadPart; + else + return av_rescale(now.QuadPart, TIMESPAN_RES64, freq); +} + +static int64_t dup_interval(GfxCaptureContext *cctx) +{ + return av_rescale(TIMESPAN_RES64, cctx->frame_rate.den, cctx->frame_rate.num); +} + +// frame->pts is in QPC 100 ns units, as SystemRelativeTime +static int output_frame(AVFilterLink *outlink, AVFrame *frame) +{ + GfxCaptureContextCpp *ctx = CCTX(outlink->src->priv)->ctx; + + // a captured frame can carry a time slightly before an already sent repeat + if (ctx->last_pts != AV_NOPTS_VALUE && frame->pts <= ctx->last_pts) + frame->pts = ctx->last_pts + 1; + + ctx->last_pts = frame->pts; + + if (ctx->first_pts == AV_NOPTS_VALUE) + ctx->first_pts = frame->pts; + frame->pts -= ctx->first_pts; + + return ff_filter_frame(outlink, frame); +} + static int process_frame_if_exists(AVFilterLink *outlink) { AVFilterContext *avctx = outlink->src; @@ -1434,29 +1523,61 @@ static int process_frame_if_exists(AVFilterLink *outlink) if (ret < 0) return ret; - frame->sample_aspect_ratio = AVRational{1, 1}; + set_frame_props(ctx, frame); - if (ctx->frames_ctx->sw_format == AV_PIX_FMT_RGBAF16) { - // According to MSDN, all floating point formats contain sRGB image data with linear 1.0 gamma. - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_LINEAR; - frame->colorspace = AVCOL_SPC_RGB; - } else { - // According to MSDN, all integer formats contain sRGB image data - frame->color_range = AVCOL_RANGE_JPEG; - frame->color_primaries = AVCOL_PRI_BT709; - frame->color_trc = AVCOL_TRC_IEC61966_2_1; - frame->colorspace = AVCOL_SPC_RGB; + if (cctx->dup_frames) { + // Half an interval of slack, so a source running at max_framerate does not get + // repeats interleaved on timer jitter. + ctx->next_pts = qpc_now() + dup_interval(cctx) * 3 / 2; + + if (!ctx->last_frame && !(ctx->last_frame = av_frame_alloc())) { + av_frame_free(&frame); + return AVERROR(ENOMEM); + } + ret = av_frame_replace(ctx->last_frame, frame); + if (ret < 0) { + av_frame_free(&frame); + return ret; + } } - ctx->last_pts = frame->pts; + return output_frame(outlink, frame); +} - if (!ctx->first_pts) - ctx->first_pts = frame->pts; - frame->pts -= ctx->first_pts; +// dup_frames: no new frame arrived in time, repeat the last one, or send black before the first. +static int output_dup_frame(AVFilterLink *outlink) +{ + AVFilterContext *avctx = outlink->src; + GfxCaptureContext *cctx = CCTX(avctx->priv); + GfxCaptureContextCpp *ctx = cctx->ctx; + AVFrame *frame; + int ret; - return ff_filter_frame(outlink, frame); + if (!ctx->last_frame) { + ctx->last_frame = ff_get_video_buffer(outlink, cctx->canvas_width, cctx->canvas_height); + if (!ctx->last_frame) + return AVERROR(ENOMEM); + ret = clear_frame(avctx, ctx->last_frame); + if (ret < 0) { + av_frame_free(&ctx->last_frame); + return ret; + } + set_frame_props(ctx, ctx->last_frame); + } + + frame = av_frame_clone(ctx->last_frame); + if (!frame) + return AVERROR(ENOMEM); + + // Stamp with the deadline and advance from it, so timer overshoot shows up + // neither in the timestamps nor in the rate. Restart from now after a stall. + int64_t now = qpc_now(); + frame->pts = ctx->next_pts; + ctx->next_pts += dup_interval(cctx); + if (ctx->next_pts <= now) + ctx->next_pts = now + dup_interval(cctx); + + return output_frame(outlink, frame); } static int gfxcapture_activate(AVFilterContext *avctx) @@ -1485,14 +1606,30 @@ static int gfxcapture_activate(AVFilterContext *avctx) std::unique_lock frame_lock(wgctx->frame_arrived_mutex); if (wgctx->window_closed && wgctx->frame_seq == last_seq) { - ff_outlink_set_status(outlink, AVERROR_EOF, ctx->last_pts - ctx->first_pts + 1); + ff_outlink_set_status(outlink, AVERROR_EOF, + ctx->last_pts == AV_NOPTS_VALUE ? AV_NOPTS_VALUE + : ctx->last_pts - ctx->first_pts + 1); break; } - if (!wgctx->frame_arrived_cond.wait_for(frame_lock, std::chrono::seconds(1), [&]() { + int64_t timeout = TIMESPAN_RES64; + if (cctx->dup_frames) { + if (ctx->next_pts == AV_NOPTS_VALUE) + ctx->next_pts = qpc_now() + dup_interval(cctx); + timeout = av_clip64(ctx->next_pts - qpc_now(), 0, TIMESPAN_RES64); + } + + if (wgctx->frame_arrived_cond.wait_for(frame_lock, std::chrono::nanoseconds(timeout * 100), [&]() { return wgctx->frame_seq != last_seq || wgctx->window_closed; })) - break; + continue; + + if (cctx->dup_frames && qpc_now() >= ctx->next_pts) { + frame_lock.unlock(); + return output_dup_frame(outlink); + } + + break; } return 0;