Separate history
Some checks failed
CI / Native Windows Build And Tests (push) Has been cancelled
CI / React UI Build (push) Has been cancelled
CI / Windows Release Package (push) Has been cancelled

This commit is contained in:
2026-05-06 10:14:55 +10:00
parent 9e480db31c
commit 8c8028dd1f
10 changed files with 723 additions and 596 deletions

View File

@@ -58,8 +58,6 @@
#include <cctype>
#include <limits>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <vector>
@@ -116,28 +114,7 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
mDeckLinkSupportsExternalKeying(false),
mDeckLinkKeyerInterfaceAvailable(false),
mDeckLinkExternalKeyingActive(false),
mFastTransferExtensionAvailable(false),
mCaptureTexture(0),
mDecodedTexture(0),
mLayerTempTexture(0),
mFBOTexture(0),
mOutputTexture(0),
mUnpinnedTextureBuffer(0),
mDecodeFrameBuf(0),
mLayerTempFrameBuf(0),
mIdFrameBuf(0),
mOutputFrameBuf(0),
mIdColorBuf(0),
mIdDepthBuf(0),
mFullscreenVAO(0),
mGlobalParamsUBO(0),
mDecodeProgram(0),
mDecodeVertexShader(0),
mDecodeFragmentShader(0),
mGlobalParamsUBOSize(0),
mViewWidth(0),
mViewHeight(0),
mTemporalHistoryNeedsReset(true)
mRenderer(std::make_unique<OpenGLRenderer>())
{
InitializeCriticalSection(&pMutex);
mRuntimeHost = std::make_unique<RuntimeHost>();
@@ -201,38 +178,7 @@ OpenGLComposite::~OpenGLComposite()
mPlayoutAllocator = NULL;
}
if (mFullscreenVAO != 0)
glDeleteVertexArrays(1, &mFullscreenVAO);
if (mGlobalParamsUBO != 0)
glDeleteBuffers(1, &mGlobalParamsUBO);
if (mDecodeFrameBuf != 0)
glDeleteFramebuffers(1, &mDecodeFrameBuf);
if (mLayerTempFrameBuf != 0)
glDeleteFramebuffers(1, &mLayerTempFrameBuf);
if (mIdFrameBuf != 0)
glDeleteFramebuffers(1, &mIdFrameBuf);
if (mIdColorBuf != 0)
glDeleteRenderbuffers(1, &mIdColorBuf);
if (mIdDepthBuf != 0)
glDeleteRenderbuffers(1, &mIdDepthBuf);
if (mCaptureTexture != 0)
glDeleteTextures(1, &mCaptureTexture);
if (mDecodedTexture != 0)
glDeleteTextures(1, &mDecodedTexture);
if (mLayerTempTexture != 0)
glDeleteTextures(1, &mLayerTempTexture);
if (mFBOTexture != 0)
glDeleteTextures(1, &mFBOTexture);
if (mOutputTexture != 0)
glDeleteTextures(1, &mOutputTexture);
if (mOutputFrameBuf != 0)
glDeleteFramebuffers(1, &mOutputFrameBuf);
if (mUnpinnedTextureBuffer != 0)
glDeleteBuffers(1, &mUnpinnedTextureBuffer);
destroyTemporalHistoryResources();
destroyLayerPrograms();
destroyDecodeShaderProgram();
mRenderer->DestroyResources();
if (mOscServer)
mOscServer->Stop();
if (mControlServer)
@@ -418,7 +364,7 @@ bool OpenGLComposite::InitDeckLink()
}
if (mInputFrameWidth != mOutputFrameWidth || mInputFrameHeight != mOutputFrameHeight)
{
mFastTransferExtensionAvailable = false;
mRenderer->mFastTransferExtensionAvailable = false;
OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n");
}
@@ -451,10 +397,10 @@ bool OpenGLComposite::InitDeckLink()
else
resizeWindow(mOutputFrameWidth / 2, mOutputFrameHeight / 2);
if (mFastTransferExtensionAvailable)
if (mRenderer->mFastTransferExtensionAvailable)
{
// Initialize fast video frame transfers
if (! VideoFrameTransfer::initialize(mInputFrameWidth, mInputFrameHeight, mCaptureTexture, mOutputTexture))
if (! VideoFrameTransfer::initialize(mInputFrameWidth, mInputFrameHeight, mRenderer->mCaptureTexture, mRenderer->mOutputTexture))
{
MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK);
goto error;
@@ -661,44 +607,7 @@ void OpenGLComposite::paintGL()
return;
}
// The DeckLink API provides IDeckLinkGLScreenPreviewHelper as a convenient way to view the playout video frames
// in a window. However, it performs a copy from host memory to the GPU which is wasteful in this case since
// we already have the rendered frame to be played out sitting in the GPU in the mIdFrameBuf frame buffer.
// Copy the off-screen frame buffer to the on-screen frame buffer while preserving the
// incoming video aspect ratio. Any extra window area is cleared to black.
int destWidth = mViewWidth;
int destHeight = mViewHeight;
int destX = 0;
int destY = 0;
if (mOutputFrameWidth > 0 && mOutputFrameHeight > 0 && mViewWidth > 0 && mViewHeight > 0)
{
const double frameAspect = static_cast<double>(mOutputFrameWidth) / static_cast<double>(mOutputFrameHeight);
const double viewAspect = static_cast<double>(mViewWidth) / static_cast<double>(mViewHeight);
if (viewAspect > frameAspect)
{
destHeight = mViewHeight;
destWidth = static_cast<int>(destHeight * frameAspect + 0.5);
destX = (mViewWidth - destWidth) / 2;
}
else
{
destWidth = mViewWidth;
destHeight = static_cast<int>(destWidth / frameAspect + 0.5);
destY = (mViewHeight - destHeight) / 2;
}
}
glBindFramebuffer(GL_READ_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
glViewport(0, 0, mViewWidth, mViewHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glBlitFramebuffer(0, 0, mOutputFrameWidth, mOutputFrameHeight, destX, destY, destX + destWidth, destY + destHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
SwapBuffers(hGLDC);
mRenderer->PresentToWindow(hGLDC, mOutputFrameWidth, mOutputFrameHeight);
ValidateRect(hGLWnd, NULL);
LeaveCriticalSection(&pMutex);
}
@@ -707,8 +616,7 @@ void OpenGLComposite::resizeGL(WORD width, WORD height)
{
// We don't set the project or model matrices here since the window data is copied directly from
// an off-screen FBO in paintGL(). Just save the width and height for use in paintGL().
mViewWidth = width;
mViewHeight = height;
mRenderer->ResizeView(width, height);
}
void OpenGLComposite::resizeWindow(int width, int height)
@@ -753,132 +661,13 @@ bool OpenGLComposite::InitOpenGLState()
}
resetTemporalHistoryState();
glClearColor( 0.0f, 0.0f, 0.0f, 0.5f ); // Black background
glDisable(GL_DEPTH_TEST);
if (! mFastTransferExtensionAvailable)
std::string rendererError;
if (!mRenderer->InitializeResources(mInputFrameWidth, mInputFrameHeight, mOutputFrameWidth, mOutputFrameHeight, rendererError))
{
glGenBuffers(1, &mUnpinnedTextureBuffer);
}
// Setup the texture which will hold the captured video frame pixels
glGenTextures(1, &mCaptureTexture);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
// Parameters to control how texels are sampled from the texture
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// Create texture with empty data, we will update it using glTexSubImage2D each frame.
// The captured video is YCbCr 4:2:2 packed into a UYVY macropixel. OpenGL has no YCbCr format
// so treat it as RGBA 4:4:4:4 by halving the width and using GL_RGBA internal format.
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth / 2, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mDecodedTexture);
glBindTexture(GL_TEXTURE_2D, mDecodedTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mLayerTempTexture);
glBindTexture(GL_TEXTURE_2D, mLayerTempTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
// Create Frame Buffer Object (FBO) to perform off-screen rendering of scene.
// This allows the render to be done on a framebuffer with width and height exactly matching the video format.
glGenFramebuffers(1, &mDecodeFrameBuf);
glGenFramebuffers(1, &mLayerTempFrameBuf);
glGenFramebuffers(1, &mIdFrameBuf);
glGenFramebuffers(1, &mOutputFrameBuf);
glGenRenderbuffers(1, &mIdColorBuf);
glGenRenderbuffers(1, &mIdDepthBuf);
glGenVertexArrays(1, &mFullscreenVAO);
glGenBuffers(1, &mGlobalParamsUBO);
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mDecodedTexture, 0);
GLenum glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize decode framebuffer."), _T("OpenGL initialization error."), MB_OK);
MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK);
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mLayerTempFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mLayerTempTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize layer framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
// Texture for FBO
glGenTextures(1, &mFBOTexture);
glBindTexture(GL_TEXTURE_2D, mFBOTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
// Attach a depth buffer
glBindRenderbuffer(GL_RENDERBUFFER, mIdDepthBuf);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, mInputFrameWidth, mInputFrameHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER, mIdDepthBuf);
// Attach the texture which stores the playback image
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mFBOTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glGenTextures(1, &mOutputTexture);
glBindTexture(GL_TEXTURE_2D, mOutputTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mOutputFrameWidth, mOutputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mOutputTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize output framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glBindTexture(GL_TEXTURE_2D, 0);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindVertexArray(mFullscreenVAO);
glBindVertexArray(0);
glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO);
glBufferData(GL_UNIFORM_BUFFER, 1024, NULL, GL_DYNAMIC_DRAW);
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO);
glBindBuffer(GL_UNIFORM_BUFFER, 0);
broadcastRuntimeState();
return true;
}
@@ -913,10 +702,10 @@ void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bo
wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread
if (mFastTransferExtensionAvailable)
if (mRenderer->mFastTransferExtensionAvailable)
{
CComQIPtr<PinnedMemoryAllocator, &IID_PinnedMemoryAllocator> allocator(inputFrameBuffer);
if (!allocator || !allocator->transferFrame(videoPixels, mCaptureTexture))
if (!allocator || !allocator->transferFrame(videoPixels, mRenderer->mCaptureTexture))
OutputDebugStringA("Capture: transferFrame() failed\n");
allocator->waitForTransferComplete(videoPixels);
@@ -924,9 +713,9 @@ void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bo
else
{
// Use a straightforward texture buffer
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mUnpinnedTextureBuffer);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer->mUnpinnedTextureBuffer);
glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture);
// NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mInputFrameWidth / 2, mInputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
@@ -959,16 +748,16 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
// Draw the effect output to the off-screen framebuffer.
const auto renderStartTime = std::chrono::steady_clock::now();
if (mFastTransferExtensionAvailable)
if (mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU);
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf);
renderEffect();
glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mIdFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glFlush();
if (mFastTransferExtensionAvailable)
if (mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU);
const auto renderEndTime = std::chrono::steady_clock::now();
if (mRuntimeHost)
@@ -999,13 +788,13 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
void* pFrame;
outputVideoFrameBuffer->GetBytes(&pFrame);
if (mFastTransferExtensionAvailable)
if (mRenderer->mFastTransferExtensionAvailable)
{
// Finished with mCaptureTexture
// Finished with mRenderer->mCaptureTexture
if (!mHasNoInputSource)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
if (! mPlayoutAllocator->transferFrame(pFrame, mOutputTexture))
if (! mPlayoutAllocator->transferFrame(pFrame, mRenderer->mOutputTexture))
OutputDebugStringA("Playback: transferFrame() failed\n");
paintGL();
@@ -1015,7 +804,7 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
}
else
{
glBindFramebuffer(GL_READ_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glReadPixels(0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame);
paintGL();
}
@@ -1342,7 +1131,7 @@ bool OpenGLComposite::compileLayerPrograms(int errorMessageSize, char* errorMess
}
destroyLayerPrograms();
mLayerPrograms.swap(newPrograms);
mRenderer->mLayerPrograms.swap(newPrograms);
if (mRuntimeHost)
{
@@ -1393,57 +1182,20 @@ bool OpenGLComposite::compileDecodeShader(int errorMessageSize, char* errorMessa
}
destroyDecodeShaderProgram();
mDecodeProgram = newProgram.release();
mDecodeVertexShader = newVertexShader.release();
mDecodeFragmentShader = newFragmentShader.release();
mRenderer->mDecodeProgram = newProgram.release();
mRenderer->mDecodeVertexShader = newVertexShader.release();
mRenderer->mDecodeFragmentShader = newFragmentShader.release();
return true;
}
void OpenGLComposite::destroySingleLayerProgram(LayerProgram& layerProgram)
{
for (LayerProgram::TextureBinding& textureBinding : layerProgram.textureBindings)
{
if (textureBinding.texture != 0)
{
glDeleteTextures(1, &textureBinding.texture);
textureBinding.texture = 0;
}
}
layerProgram.textureBindings.clear();
for (LayerProgram::TextBinding& textBinding : layerProgram.textBindings)
{
if (textBinding.texture != 0)
{
glDeleteTextures(1, &textBinding.texture);
textBinding.texture = 0;
}
}
layerProgram.textBindings.clear();
if (layerProgram.program != 0)
{
glDeleteProgram(layerProgram.program);
layerProgram.program = 0;
}
if (layerProgram.fragmentShader != 0)
{
glDeleteShader(layerProgram.fragmentShader);
layerProgram.fragmentShader = 0;
}
if (layerProgram.vertexShader != 0)
{
glDeleteShader(layerProgram.vertexShader);
layerProgram.vertexShader = 0;
}
mRenderer->DestroySingleLayerProgram(layerProgram);
}
void OpenGLComposite::destroyLayerPrograms()
{
for (LayerProgram& layerProgram : mLayerPrograms)
destroySingleLayerProgram(layerProgram);
mLayerPrograms.clear();
mRenderer->DestroyLayerPrograms();
}
bool OpenGLComposite::loadTextureAsset(const ShaderTextureAsset& textureAsset, GLuint& textureId, std::string& error)
@@ -1508,241 +1260,40 @@ void OpenGLComposite::bindLayerTextureAssets(const LayerProgram& layerProgram)
void OpenGLComposite::destroyDecodeShaderProgram()
{
if (mDecodeProgram != 0)
{
glDeleteProgram(mDecodeProgram);
mDecodeProgram = 0;
}
if (mDecodeFragmentShader != 0)
{
glDeleteShader(mDecodeFragmentShader);
mDecodeFragmentShader = 0;
}
if (mDecodeVertexShader != 0)
{
glDeleteShader(mDecodeVertexShader);
mDecodeVertexShader = 0;
}
mRenderer->DestroyDecodeShaderProgram();
}
bool OpenGLComposite::validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const
{
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
unsigned requiredUnits = kSourceHistoryTextureUnitBase;
for (const RuntimeRenderState& state : layerStates)
{
unsigned textTextureCount = 0;
for (const ShaderParameterDefinition& definition : state.parameterDefinitions)
{
if (definition.type == ShaderParameterType::Text)
++textTextureCount;
}
const unsigned totalShaderTextures = static_cast<unsigned>(state.textureAssets.size()) + textTextureCount;
const unsigned layerRequiredUnits = kSourceHistoryTextureUnitBase + (state.isTemporal ? historyCap + historyCap : 0u) + totalShaderTextures;
if (layerRequiredUnits > requiredUnits)
requiredUnits = layerRequiredUnits;
}
GLint maxTextureUnits = 0;
glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits);
const unsigned availableUnits = maxTextureUnits > 0 ? static_cast<unsigned>(maxTextureUnits) : 0u;
if (requiredUnits > availableUnits)
{
std::ostringstream message;
message << "The current history and shader texture asset configuration requires " << requiredUnits
<< " fragment texture units, but only " << maxTextureUnits << " are available.";
error = message.str();
return false;
}
return true;
}
bool OpenGLComposite::createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error)
{
destroyHistoryRing(ring);
ring.effectiveLength = effectiveLength;
ring.historySource = historySource;
if (effectiveLength == 0)
return true;
ring.slots.resize(effectiveLength);
for (HistorySlot& slot : ring.slots)
{
glGenTextures(1, &slot.texture);
glBindTexture(GL_TEXTURE_2D, slot.texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glGenFramebuffers(1, &slot.framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, slot.framebuffer);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, slot.texture, 0);
const GLenum framebufferStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (framebufferStatus != GL_FRAMEBUFFER_COMPLETE)
{
error = "Failed to initialize a temporal history framebuffer.";
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
destroyHistoryRing(ring);
return false;
}
}
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
return true;
}
void OpenGLComposite::destroyHistoryRing(HistoryRing& ring)
{
for (HistorySlot& slot : ring.slots)
{
if (slot.framebuffer != 0)
glDeleteFramebuffers(1, &slot.framebuffer);
if (slot.texture != 0)
glDeleteTextures(1, &slot.texture);
slot.framebuffer = 0;
slot.texture = 0;
}
ring.slots.clear();
ring.nextWriteIndex = 0;
ring.filledCount = 0;
ring.effectiveLength = 0;
ring.historySource = TemporalHistorySource::None;
}
void OpenGLComposite::destroyTemporalHistoryResources()
{
destroyHistoryRing(mSourceHistoryRing);
for (auto& historyEntry : mPreLayerHistoryByLayerId)
destroyHistoryRing(historyEntry.second);
mPreLayerHistoryByLayerId.clear();
return mRenderer->mTemporalHistory.ValidateTextureUnitBudget(layerStates, historyCap, error);
}
void OpenGLComposite::resetTemporalHistoryState()
{
mSourceHistoryRing.nextWriteIndex = 0;
mSourceHistoryRing.filledCount = 0;
for (auto& historyEntry : mPreLayerHistoryByLayerId)
{
historyEntry.second.nextWriteIndex = 0;
historyEntry.second.filledCount = 0;
}
mTemporalHistoryNeedsReset = false;
mRenderer->mTemporalHistory.ResetState();
}
bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error)
{
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
const bool sourceHistoryNeeded = std::any_of(layerStates.begin(), layerStates.end(),
[](const RuntimeRenderState& state) { return state.isTemporal && state.effectiveTemporalHistoryLength > 0; });
const unsigned sourceHistoryLength = sourceHistoryNeeded ? historyCap : 0;
if (mSourceHistoryRing.effectiveLength != sourceHistoryLength)
{
if (!createHistoryRing(mSourceHistoryRing, sourceHistoryLength, TemporalHistorySource::Source, error))
return false;
mTemporalHistoryNeedsReset = true;
}
std::set<std::string> requiredPreLayerIds;
for (const RuntimeRenderState& state : layerStates)
{
if (!state.isTemporal || state.temporalHistorySource != TemporalHistorySource::PreLayerInput)
continue;
requiredPreLayerIds.insert(state.layerId);
auto historyIt = mPreLayerHistoryByLayerId.find(state.layerId);
if (historyIt == mPreLayerHistoryByLayerId.end() || historyIt->second.effectiveLength != state.effectiveTemporalHistoryLength)
{
HistoryRing replacement;
if (!createHistoryRing(replacement, state.effectiveTemporalHistoryLength, TemporalHistorySource::PreLayerInput, error))
return false;
mPreLayerHistoryByLayerId[state.layerId] = std::move(replacement);
mTemporalHistoryNeedsReset = true;
}
}
for (auto it = mPreLayerHistoryByLayerId.begin(); it != mPreLayerHistoryByLayerId.end();)
{
if (requiredPreLayerIds.find(it->first) == requiredPreLayerIds.end())
{
destroyHistoryRing(it->second);
it = mPreLayerHistoryByLayerId.erase(it);
mTemporalHistoryNeedsReset = true;
}
else
{
++it;
}
}
if (mTemporalHistoryNeedsReset)
resetTemporalHistoryState();
return true;
}
void OpenGLComposite::pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring)
{
if (ring.effectiveLength == 0 || ring.slots.empty())
return;
HistorySlot& targetSlot = ring.slots[ring.nextWriteIndex];
glBindFramebuffer(GL_READ_FRAMEBUFFER, sourceFramebuffer);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, targetSlot.framebuffer);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
ring.nextWriteIndex = (ring.nextWriteIndex + 1) % ring.slots.size();
ring.filledCount = std::min<std::size_t>(ring.filledCount + 1, ring.slots.size());
}
GLuint OpenGLComposite::resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const
{
if (ring.filledCount == 0 || ring.slots.empty())
return fallbackTexture;
const std::size_t clampedOffset = std::min<std::size_t>(framesAgo, ring.filledCount - 1);
const std::size_t newestIndex = (ring.nextWriteIndex + ring.slots.size() - 1) % ring.slots.size();
const std::size_t slotIndex = (newestIndex + ring.slots.size() - clampedOffset) % ring.slots.size();
return ring.slots[slotIndex].texture != 0 ? ring.slots[slotIndex].texture : fallbackTexture;
return mRenderer->mTemporalHistory.EnsureResources(layerStates, historyCap, mInputFrameWidth, mInputFrameHeight, error);
}
unsigned OpenGLComposite::sourceHistoryAvailableCount() const
{
return static_cast<unsigned>(mSourceHistoryRing.filledCount);
return mRenderer->mTemporalHistory.SourceAvailableCount();
}
unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const
{
auto it = mPreLayerHistoryByLayerId.find(layerId);
if (it == mPreLayerHistoryByLayerId.end())
return 0;
return static_cast<unsigned>(it->second.filledCount);
return mRenderer->mTemporalHistory.AvailableCountForLayer(layerId);
}
void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture)
{
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
for (unsigned index = 0; index < historyCap; ++index)
{
glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index);
glBindTexture(GL_TEXTURE_2D, resolveHistoryTexture(mSourceHistoryRing, currentSourceTexture, index));
}
const GLuint temporalBase = kSourceHistoryTextureUnitBase + historyCap;
const HistoryRing* temporalRing = nullptr;
auto it = mPreLayerHistoryByLayerId.find(state.layerId);
if (it != mPreLayerHistoryByLayerId.end())
temporalRing = &it->second;
for (unsigned index = 0; index < historyCap; ++index)
{
glActiveTexture(GL_TEXTURE0 + temporalBase + index);
glBindTexture(GL_TEXTURE_2D, temporalRing ? resolveHistoryTexture(*temporalRing, currentSourceTexture, index) : currentSourceTexture);
}
glActiveTexture(GL_TEXTURE0);
mRenderer->mTemporalHistory.BindSamplers(state, currentSourceTexture, historyCap);
}
void OpenGLComposite::renderEffect()
@@ -1750,9 +1301,9 @@ void OpenGLComposite::renderEffect()
PollRuntimeChanges();
const bool hasInputSource = !mHasNoInputSource;
if (hasInputSource && mFastTransferExtensionAvailable)
if (hasInputSource && mRenderer->mFastTransferExtensionAvailable)
{
// Signal that we're about to draw using mCaptureTexture onto mFBOTexture.
// Signal that we're about to draw using mRenderer->mCaptureTexture onto mRenderer->mFBOTexture.
VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU);
}
@@ -1764,43 +1315,39 @@ void OpenGLComposite::renderEffect()
}
else
{
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glViewport(0, 0, mInputFrameWidth, mInputFrameHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
}
const std::vector<RuntimeRenderState> layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mInputFrameWidth, mInputFrameHeight) : std::vector<RuntimeRenderState>();
if (layerStates.empty() || mLayerPrograms.empty())
if (layerStates.empty() || mRenderer->mLayerPrograms.empty())
{
glBindFramebuffer(GL_READ_FRAMEBUFFER, mDecodeFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mIdFrameBuf);
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mIdFrameBuf);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf);
}
else
{
GLuint sourceTexture = mDecodedTexture;
GLuint sourceFrameBuffer = mDecodeFrameBuf;
for (std::size_t index = 0; index < layerStates.size() && index < mLayerPrograms.size(); ++index)
GLuint sourceTexture = mRenderer->mDecodedTexture;
GLuint sourceFrameBuffer = mRenderer->mDecodeFrameBuf;
for (std::size_t index = 0; index < layerStates.size() && index < mRenderer->mLayerPrograms.size(); ++index)
{
const std::size_t remaining = layerStates.size() - index;
const bool writeToMain = (remaining % 2) == 1;
renderShaderProgram(sourceTexture, writeToMain ? mIdFrameBuf : mLayerTempFrameBuf, mLayerPrograms[index], layerStates[index]);
renderShaderProgram(sourceTexture, writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf, mRenderer->mLayerPrograms[index], layerStates[index]);
if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput)
{
auto historyIt = mPreLayerHistoryByLayerId.find(layerStates[index].layerId);
if (historyIt != mPreLayerHistoryByLayerId.end())
pushFramebufferToHistoryRing(sourceFrameBuffer, historyIt->second);
}
sourceTexture = writeToMain ? mFBOTexture : mLayerTempTexture;
sourceFrameBuffer = writeToMain ? mIdFrameBuf : mLayerTempFrameBuf;
mRenderer->mTemporalHistory.PushPreLayerFramebuffer(layerStates[index].layerId, sourceFrameBuffer, mInputFrameWidth, mInputFrameHeight);
sourceTexture = writeToMain ? mRenderer->mFBOTexture : mRenderer->mLayerTempTexture;
sourceFrameBuffer = writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf;
}
}
pushFramebufferToHistoryRing(mDecodeFrameBuf, mSourceHistoryRing);
mRenderer->mTemporalHistory.PushSourceFramebuffer(mRenderer->mDecodeFrameBuf, mInputFrameWidth, mInputFrameHeight);
if (hasInputSource && mFastTransferExtensionAvailable)
if (hasInputSource && mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
}
@@ -1820,7 +1367,7 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
glBindTexture(GL_TEXTURE_2D, sourceTexture);
bindHistorySamplers(state, sourceTexture);
bindLayerTextureAssets(layerProgram);
glBindVertexArray(mFullscreenVAO);
glBindVertexArray(mRenderer->mFullscreenVAO);
glUseProgram(layerProgram.program);
updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId));
glDrawArrays(GL_TRIANGLES, 0, 3);
@@ -1847,16 +1394,16 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
void OpenGLComposite::renderDecodePass()
{
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glViewport(0, 0, mInputFrameWidth, mInputFrameHeight);
glClear(GL_COLOR_BUFFER_BIT);
glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
glBindVertexArray(mFullscreenVAO);
glUseProgram(mDecodeProgram);
glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture);
glBindVertexArray(mRenderer->mFullscreenVAO);
glUseProgram(mRenderer->mDecodeProgram);
const GLint packedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uPackedVideoResolution");
const GLint decodedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uDecodedVideoResolution");
const GLint packedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uPackedVideoResolution");
const GLint decodedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uDecodedVideoResolution");
if (packedResolutionLocation >= 0)
glUniform2f(packedResolutionLocation, static_cast<float>(mInputFrameWidth / 2), static_cast<float>(mInputFrameHeight));
if (decodedResolutionLocation >= 0)
@@ -1979,17 +1526,17 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state,
buffer.resize(AlignStd140(buffer.size(), 16), 0);
glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO);
if (mGlobalParamsUBOSize != static_cast<GLsizeiptr>(buffer.size()))
glBindBuffer(GL_UNIFORM_BUFFER, mRenderer->mGlobalParamsUBO);
if (mRenderer->mGlobalParamsUBOSize != static_cast<GLsizeiptr>(buffer.size()))
{
glBufferData(GL_UNIFORM_BUFFER, static_cast<GLsizeiptr>(buffer.size()), buffer.data(), GL_DYNAMIC_DRAW);
mGlobalParamsUBOSize = static_cast<GLsizeiptr>(buffer.size());
mRenderer->mGlobalParamsUBOSize = static_cast<GLsizeiptr>(buffer.size());
}
else
{
glBufferSubData(GL_UNIFORM_BUFFER, 0, static_cast<GLsizeiptr>(buffer.size()), buffer.data());
}
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO);
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mRenderer->mGlobalParamsUBO);
glBindBuffer(GL_UNIFORM_BUFFER, 0);
return true;
@@ -1997,12 +1544,13 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state,
bool OpenGLComposite::CheckOpenGLExtensions()
{
mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable();
mRenderer->mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable();
if (!mFastTransferExtensionAvailable)
if (!mRenderer->mFastTransferExtensionAvailable)
OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n");
return true;
}
////////////////////////////////////////////