Fix Xbox One performance regressions. Refactor render loop wait behavior and re-enable explicit ffmpeg/D3D locking.

Author: andygrundman

State/Stats.cpp

@@ -276,7 +276,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha
 	if (stats.receivedFps > 0) {
 		ret = snprintf(&output[offset],
 						length - offset,
-						"Video stream: %dx%d %.2f FPS (Codec: %s)\n",
+						"Video stream: %dx%d %.2f FPS (%s)\n",
 						ffmpeg.width,
 						ffmpeg.height,
 						stats.totalFps,
@@ -355,7 +355,7 @@ void Stats::formatVideoStats(DX::StepTimer const& timer, VIDEO_STATS& stats, cha
 					   "Average network latency: %s\n"
 					   "Average reassembly/decoding time: %.2f/%.2f ms\n"
 					   "Average frames in queue: %.1f\n"
-					   "Average frame queue/render/present time: %.2f/%.2f/%.2f ms\n",
+					   "Average frame queue/render/present: %.2f/%.2f/%.2f ms\n",
 					   stats.totalFrames ? (double)stats.networkDroppedFrames / stats.totalFrames * 100 : 0.0f,
 					   stats.totalFrames ? (double)stats.pacerDroppedFrames / stats.totalFrames * 100 : 0.0f,
 					   rttString,

Streaming/FFmpegDecoder.cpp

@@ -58,6 +58,16 @@ namespace moonlight_xbox_dx {
 		m_LastFrameNumber(0) {
 	}
 
+	void lock_context(void *user) {
+		auto me = (FFMpegDecoder*)user;
+		me->m_mutex.lock();
+	}
+
+	void unlock_context(void *user) {
+		auto me = (FFMpegDecoder*)user;
+		me->m_mutex.unlock();
+	}
+
 	void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) {
 		char lineBuffer[1024];
 		static int printPrefix = 1;
@@ -75,7 +85,7 @@ namespace moonlight_xbox_dx {
 		Utils::Logf(shouldPrefixThisMessage ? "[ffmpeg] %s" : "%s", lineBuffer);
 	}
 
-	void FFMpegDecoder::CompleteInitialization(const std::shared_ptr<DX::DeviceResources>& res, STREAM_CONFIGURATION *config) {
+    void FFMpegDecoder::CompleteInitialization(const std::shared_ptr<DX::DeviceResources>& res, STREAM_CONFIGURATION *config) {
 		m_deviceResources = res;
 		Pacer::instance().init(res, config->fps, res->GetRefreshRate());
 	}
@@ -124,6 +134,9 @@ namespace moonlight_xbox_dx {
 		d3d11va_device_ctx = reinterpret_cast<AVD3D11VADeviceContext*>(device_ctx->hwctx);
 		d3d11va_device_ctx->device = m_deviceResources->GetD3DDevice();
 		d3d11va_device_ctx->device_context = m_deviceResources->GetD3DDeviceContext();
+		d3d11va_device_ctx->lock = lock_context;
+		d3d11va_device_ctx->unlock = unlock_context;
+		d3d11va_device_ctx->lock_ctx = this;
 		int err2;
 		if ((err2 = av_hwdevice_ctx_init(hw_device_ctx)) < 0) {
 			Utils::Logf("Failed to create specified DirectX Video device: %d\n", err2);
@@ -139,7 +152,7 @@ namespace moonlight_xbox_dx {
 		decoder_ctx->width = width;
 		decoder_ctx->height = height;
 
-	    int err = avcodec_open2(decoder_ctx, decoder, NULL);
+		int err = avcodec_open2(decoder_ctx, decoder, NULL);
 		if (err < 0) {
 			char msg[2048];
 			sprintf(msg, "Failed to create FFMpeg Codec: %d\n", err);
@@ -157,13 +170,6 @@ namespace moonlight_xbox_dx {
 			return -1;
 		}
 
-		// Put D3D11 in multithread-friendly mode
-		ID3D11Multithread *pMultithread = nullptr;
-		HRESULT hr = d3d11va_device_ctx->device->QueryInterface(__uuidof(ID3D11Multithread), (void **)&pMultithread);
-		if (SUCCEEDED(hr)) {
-			pMultithread->SetMultithreadProtected(TRUE);
-			pMultithread->Release();
-		}
 		return 0;
 	}
 

Streaming/FFmpegDecoder.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <atomic>
+#include <mutex>
 #include <queue>
 #include "../Common/StepTimer.h"
 #include "Pacer.h"
@@ -17,9 +18,9 @@ extern "C" {
 #define MAX_BUFFER 1024 * 1024
 
 typedef struct MLFrameData {
-	int64_t decodeEndQpc;        // when we finished decoding
-	int64_t presentTargetQpc;    // timestamp when frame should be presented (slightly earlier than vsync)
-	int64_t presentVsyncQpc;     // hard vsync deadline
+	int64_t decodeEndQpc;     // when we finished decoding
+	int64_t presentTargetQpc; // timestamp when frame should be presented (slightly earlier than vsync)
+	int64_t presentVsyncQpc;  // hard vsync deadline
 } MLFrameData;
 
 namespace moonlight_xbox_dx {
@@ -36,6 +37,28 @@ class FFMpegDecoder {
 	static FFMpegDecoder *getInstance();
 	static DECODER_RENDERER_CALLBACKS getDecoder();
 	int videoFormat, width, height;
+	std::recursive_mutex m_mutex;
+
+	// locking helper
+	class LockGuard {
+	  public:
+		explicit LockGuard(FFMpegDecoder &ff)
+		    : m_ff(ff) {
+			m_ff.m_mutex.lock();
+		}
+		~LockGuard() {
+			m_ff.m_mutex.unlock();
+		}
+		LockGuard(const LockGuard &) = delete;
+		LockGuard &operator=(const LockGuard &) = delete;
+
+	  private:
+		FFMpegDecoder &m_ff;
+	};
+
+	[[nodiscard]] static LockGuard Lock() {
+		return LockGuard(instance());
+	}
 
   private:
 	FFMpegDecoder();

Streaming/Pacer.cpp

@@ -190,15 +190,21 @@ void Pacer::updateFrameStats() {
 
 // Main render thread
 
-void Pacer::waitForFrame(double timeoutMs) {
+void Pacer::waitForFrame(double timeoutMs, bool waitFullTime) {
 	if (!running()) return;
 
-	// mark frame begin for use by afterPresent
-	m_BeginFrameQpc = QpcNow();
-
 	// Wait for a decoded frame to be available
+	int64_t t0 = QpcNow();
 	int queueHas = 1;
 	FrameQueue::instance().waitForEnqueue(queueHas, timeoutMs);
+
+	if (waitFullTime) {
+		int64_t t1 = QpcNow();
+		timeoutMs -= QpcToMs(t1 - t0);
+		if (timeoutMs > 0.0) {
+			SleepUntilQpc(t1 + MsToQpc(timeoutMs));
+		}
+	}
 }
 
 // called by render thread
@@ -214,7 +220,9 @@ bool Pacer::renderOnMainThread(std::shared_ptr<VideoRenderer> &sceneRenderer) {
 
 	// Render it
 	FQLog("> Frame rendered [pts: %.3f]\n", frame->pts / 90.0);
-	sceneRenderer->Render(frame);
+	if (!sceneRenderer->Render(frame)) {
+		return false; // something went wrong rendering the frame
+	}
 
 	if (frame->opaque_ref) {
 		// Count time spent in FrameQueue
@@ -227,26 +235,23 @@ bool Pacer::renderOnMainThread(std::shared_ptr<VideoRenderer> &sceneRenderer) {
 }
 
 // called by render thread
-void Pacer::waitBeforePresent() {
+void Pacer::waitBeforePresent(int64_t deadline = 0) {
 	if (!running()) return;
 
 	// Wait until vsync
-	int64_t now, interval = 0;
-	int64_t target = getNextVBlankQpc(&now, &interval);
-
-	if (IsXbox() && m_StreamFps == 120 && m_RefreshRate > 119.0) {
-		// 120hz on Xbox requires us to present each frame at half-vsync intervals
-		int64_t half = interval / 2;
-		if (target - half > now) {
-			// we're currently in the first half of a vblank, sleep till the halfway mark
-			target -= half;
-		}
+	int64_t now = 0;
+	int64_t target = getNextVBlankQpc(&now);
+	if (deadline && deadline != target) {
+		// we missed the deadline
+		target = deadline;
 	}
 
 	m_LastSyncTarget.store(target, std::memory_order_release);
 
-	FQLog("waitBeforePresent(): waiting %.3fms to target %lld\n", QpcToMs(target - now), target);
-	SleepUntilQpc(target);
+	if (target > now) {
+		FQLog("waitBeforePresent(): waiting %.3fms\n", QpcToMs(target - now));
+		SleepUntilQpc(target);
+	}
 }
 
 // end main thread
@@ -267,23 +272,34 @@ void Pacer::submitFrame(AVFrame *frame) {
 
 // Caller often needs now and the vsync interval, since this needs locking
 // the logic is confined to this function.
-int64_t Pacer::getNextVBlankQpc(int64_t *now, int64_t *interval) {
+int64_t Pacer::getNextVBlankQpc(int64_t *now) {
 	std::scoped_lock<std::mutex> lock(m_FrameStatsLock);
+	int64_t target, interval = 0;
+	*now = QpcNow();
 
 	if (m_LastSyncQpc == 0 || m_VsyncIntervalQpc == 0) {
 		// Fallback until vsyncHardware spins up
-		*now = QpcNow();
 		double rr = m_RefreshRate > 0.0 ? m_RefreshRate : 60.0;
-		*interval = MsToQpc(1000.0 / rr);
-		return *now + *interval;
-	}
+		interval = MsToQpc(1000.0 / rr);
+		target = *now + interval;
+	} else {
+		interval = m_VsyncIntervalQpc;
+		int64_t next = m_LastSyncQpc;
 
-	*now = QpcNow();
-	*interval = m_VsyncIntervalQpc;
-	int64_t next = m_LastSyncQpc;
+		while (next < *now) {
+			next += interval;
+		}
+		target = next + static_cast<int64_t>(m_ewmaVsyncDriftQpc);
+	}
 
-	while (next < *now) {
-		next += *interval;
+	if (IsXbox() && m_StreamFps == 120 && m_RefreshRate > 119.0) {
+		// 120hz on Xbox requires us to present each frame at half-vsync intervals
+		int64_t half = interval / 2;
+		if (target - half > *now) {
+			// we're currently in the first half of a vblank, sleep till the halfway mark
+			target -= half;
+		}
 	}
-	return next + static_cast<int64_t>(m_ewmaVsyncDriftQpc);
+
+	return target;
 }

Streaming/Pacer.h

@@ -19,9 +19,10 @@ class Pacer {
 
 	void deinit();
 	void init(const std::shared_ptr<DX::DeviceResources> &res, int maxVideoFps, double refreshRate);
-	void waitForFrame(double timeoutMs);
+	void waitForFrame(double timeoutMs, bool waitFullTime);
 	bool renderOnMainThread(std::shared_ptr<moonlight_xbox_dx::VideoRenderer> &sceneRenderer);
-	void waitBeforePresent();
+	void waitBeforePresent(int64_t deadline);
+	int64_t getNextVBlankQpc(int64_t *now);
 	void submitFrame(AVFrame *frame);
 
   private:
@@ -39,15 +40,13 @@ class Pacer {
 
 	void vsyncHardware();
 	void updateFrameStats();
-	int64_t getNextVBlankQpc(int64_t *now, int64_t *interval);
 
 	std::shared_ptr<DX::DeviceResources> m_DeviceResources;
 	std::thread m_VsyncThread;
 	std::atomic<bool> m_Running{false};
 	std::atomic<bool> m_Stopping{false};
 	int m_StreamFps;
 	double m_RefreshRate;
-	int64_t m_BeginFrameQpc = 0;
 
 	static constexpr int VSYNC_HISTORY_SIZE = 512;
 	std::mutex m_FrameStatsLock;

Streaming/VideoRenderer.cpp

@@ -100,24 +100,34 @@ bool VideoRenderer::Render(AVFrame *frame) {
 	auto *ctx = m_deviceResources->GetD3DDeviceContext();
 	auto *dev = m_deviceResources->GetD3DDevice();
 
+	// Clear the back buffer
+	ID3D11RenderTargetView* renderTarget[] = { m_deviceResources->GetBackBufferRenderTargetView() };
+	ctx->ClearRenderTargetView(renderTarget[0], Colors::Black);
+
+	// Bind the back buffer. This needs to be done each time,
+	// because the render target view will be unbound by Present().
+	ctx->OMSetRenderTargets(1, renderTarget, nullptr);
+
 	ID3D11Texture2D *ffmpegTexture = (ID3D11Texture2D *)(frame->data[0]);
+	if (!ffmpegTexture) {
+		// This sometimes happens when reconnecting
+		return false;
+	}
 	D3D11_TEXTURE2D_DESC ffmpegDesc;
 	ffmpegTexture->GetDesc(&ffmpegDesc);
 
 	bool hasChanged = hasFrameFormatChanged(frame);
 	if (hasChanged) {
-		// Create our internal texture to copy and render
 		setupVideoTexture(ffmpegDesc);
 	}
 
+	// SRV 0 is always mapped to the video texture
+	UINT srvIndex = 0;
 	// Copy this frame into our video texture
 	ctx->CopySubresourceRegion1(m_VideoTexture.Get(), 0, 0, 0, 0,
-	                            (ID3D11Resource*)frame->data[0], (int)(intptr_t)frame->data[1],
+	                            (ID3D11Resource *)frame->data[0], (int)(intptr_t)frame->data[1],
 	                            nullptr, D3D11_COPY_DISCARD);
 
-	// SRV 0 is always mapped to the video texture
-	UINT srvIndex = 0;
-
 	// Setup shader
 	ctx->PSSetSamplers(0, 1, m_samplerState.GetAddressOf());
 	ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);