aiden/video-shader-toys
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Wiki Activity

Separate history
Some checks failed
CI / Native Windows Build And Tests (push) Has been cancelled
Details
CI / React UI Build (push) Has been cancelled
Details
CI / Windows Release Package (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Aiden
2026-05-06 10:14:55 +10:00
parent 9e480db31c
commit 8c8028dd1f
10 changed files with 723 additions and 596 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@@ -56,11 +56,15 @@ set(APP_SOURCES
"${APP_DIR}/gl/OpenGLComposite.cpp" "${APP_DIR}/gl/OpenGLComposite.cpp"
"${APP_DIR}/gl/OpenGLComposite.h" "${APP_DIR}/gl/OpenGLComposite.h"
"${APP_DIR}/gl/OpenGLCompositeRuntimeControls.cpp" "${APP_DIR}/gl/OpenGLCompositeRuntimeControls.cpp"
"${APP_DIR}/gl/OpenGLRenderer.cpp"
"${APP_DIR}/gl/OpenGLRenderer.h"
"${APP_DIR}/gl/Std140Buffer.h" "${APP_DIR}/gl/Std140Buffer.h"
"${APP_DIR}/gl/TextRasterizer.cpp" "${APP_DIR}/gl/TextRasterizer.cpp"
"${APP_DIR}/gl/TextRasterizer.h" "${APP_DIR}/gl/TextRasterizer.h"
"${APP_DIR}/gl/TextureAssetLoader.cpp" "${APP_DIR}/gl/TextureAssetLoader.cpp"
"${APP_DIR}/gl/TextureAssetLoader.h" "${APP_DIR}/gl/TextureAssetLoader.h"
"${APP_DIR}/gl/TemporalHistoryBuffers.cpp"
"${APP_DIR}/gl/TemporalHistoryBuffers.h"
"${APP_DIR}/gl/VideoFrameTransfer.cpp" "${APP_DIR}/gl/VideoFrameTransfer.cpp"
"${APP_DIR}/gl/VideoFrameTransfer.h" "${APP_DIR}/gl/VideoFrameTransfer.h"
"${APP_DIR}/LoopThroughWithOpenGLCompositing.cpp" "${APP_DIR}/LoopThroughWithOpenGLCompositing.cpp"

4
apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.vcxproj
View File

@@ -195,6 +195,8 @@
<ClCompile Include="gl\GLExtensions.cpp" /> <ClCompile Include="gl\GLExtensions.cpp" />
<ClCompile Include="LoopThroughWithOpenGLCompositing.cpp" /> <ClCompile Include="LoopThroughWithOpenGLCompositing.cpp" />
<ClCompile Include="gl\OpenGLComposite.cpp" /> <ClCompile Include="gl\OpenGLComposite.cpp" />
<ClCompile Include="gl\OpenGLRenderer.cpp" />
<ClCompile Include="gl\TemporalHistoryBuffers.cpp" />
<ClCompile Include="stdafx.cpp"> <ClCompile Include="stdafx.cpp">
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>
<PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>
@@ -208,6 +210,8 @@
<ClInclude Include="gl\GLExtensions.h" /> <ClInclude Include="gl\GLExtensions.h" />
<ClInclude Include="LoopThroughWithOpenGLCompositing.h" /> <ClInclude Include="LoopThroughWithOpenGLCompositing.h" />
<ClInclude Include="gl\OpenGLComposite.h" /> <ClInclude Include="gl\OpenGLComposite.h" />
<ClInclude Include="gl\OpenGLRenderer.h" />
<ClInclude Include="gl\TemporalHistoryBuffers.h" />
<ClInclude Include="resource.h" /> <ClInclude Include="resource.h" />
<ClInclude Include="stdafx.h" /> <ClInclude Include="stdafx.h" />
<ClInclude Include="targetver.h" /> <ClInclude Include="targetver.h" />

12
apps/LoopThroughWithOpenGLCompositing/LoopThroughWithOpenGLCompositing.vcxproj.filters
View File

@@ -27,6 +27,12 @@
<ClCompile Include="gl\OpenGLComposite.cpp"> <ClCompile Include="gl\OpenGLComposite.cpp">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="gl\OpenGLRenderer.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="gl\TemporalHistoryBuffers.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="stdafx.cpp"> <ClCompile Include="stdafx.cpp">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
@@ -47,6 +53,12 @@
<ClInclude Include="gl\OpenGLComposite.h"> <ClInclude Include="gl\OpenGLComposite.h">
<Filter>Header Files</Filter> <Filter>Header Files</Filter>
</ClInclude> </ClInclude>
<ClInclude Include="gl\OpenGLRenderer.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="gl\TemporalHistoryBuffers.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="resource.h"> <ClInclude Include="resource.h">
<Filter>Header Files</Filter> <Filter>Header Files</Filter>
</ClInclude> </ClInclude>

1
apps/LoopThroughWithOpenGLCompositing/gl/GLExtensions.h
View File

@@ -67,6 +67,7 @@
#define GL_VERTEX_SHADER 0x8B31 #define GL_VERTEX_SHADER 0x8B31
#define GL_COMPILE_STATUS 0x8B81 #define GL_COMPILE_STATUS 0x8B81
#define GL_LINK_STATUS 0x8B82 #define GL_LINK_STATUS 0x8B82
#define GL_MAX_TEXTURE_IMAGE_UNITS 0x8872
#define GL_RENDERBUFFER_EXT 0x8D41 #define GL_RENDERBUFFER_EXT 0x8D41
#define GL_FRAMEBUFFER_EXT 0x8D40 #define GL_FRAMEBUFFER_EXT 0x8D40
#define GL_FRAMEBUFFER_COMPLETE_EXT 0x8CD5 #define GL_FRAMEBUFFER_COMPLETE_EXT 0x8CD5

586
apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp
View File

@@ -58,8 +58,6 @@
#include <cctype> #include <cctype>
#include <limits> #include <limits>
#include <memory> #include <memory>
#include <set>
#include <sstream>
#include <string> #include <string>
#include <vector> #include <vector>
@@ -116,28 +114,7 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
mDeckLinkSupportsExternalKeying(false), mDeckLinkSupportsExternalKeying(false),
mDeckLinkKeyerInterfaceAvailable(false), mDeckLinkKeyerInterfaceAvailable(false),
mDeckLinkExternalKeyingActive(false), mDeckLinkExternalKeyingActive(false),
mFastTransferExtensionAvailable(false), mRenderer(std::make_unique<OpenGLRenderer>())
mCaptureTexture(0),
mDecodedTexture(0),
mLayerTempTexture(0),
mFBOTexture(0),
mOutputTexture(0),
mUnpinnedTextureBuffer(0),
mDecodeFrameBuf(0),
mLayerTempFrameBuf(0),
mIdFrameBuf(0),
mOutputFrameBuf(0),
mIdColorBuf(0),
mIdDepthBuf(0),
mFullscreenVAO(0),
mGlobalParamsUBO(0),
mDecodeProgram(0),
mDecodeVertexShader(0),
mDecodeFragmentShader(0),
mGlobalParamsUBOSize(0),
mViewWidth(0),
mViewHeight(0),
mTemporalHistoryNeedsReset(true)
{ {
InitializeCriticalSection(&pMutex); InitializeCriticalSection(&pMutex);
mRuntimeHost = std::make_unique<RuntimeHost>(); mRuntimeHost = std::make_unique<RuntimeHost>();
@@ -201,38 +178,7 @@ OpenGLComposite::~OpenGLComposite()
mPlayoutAllocator = NULL; mPlayoutAllocator = NULL;
} }
if (mFullscreenVAO != 0) mRenderer->DestroyResources();
glDeleteVertexArrays(1, &mFullscreenVAO);
if (mGlobalParamsUBO != 0)
glDeleteBuffers(1, &mGlobalParamsUBO);
if (mDecodeFrameBuf != 0)
glDeleteFramebuffers(1, &mDecodeFrameBuf);
if (mLayerTempFrameBuf != 0)
glDeleteFramebuffers(1, &mLayerTempFrameBuf);
if (mIdFrameBuf != 0)
glDeleteFramebuffers(1, &mIdFrameBuf);
if (mIdColorBuf != 0)
glDeleteRenderbuffers(1, &mIdColorBuf);
if (mIdDepthBuf != 0)
glDeleteRenderbuffers(1, &mIdDepthBuf);
if (mCaptureTexture != 0)
glDeleteTextures(1, &mCaptureTexture);
if (mDecodedTexture != 0)
glDeleteTextures(1, &mDecodedTexture);
if (mLayerTempTexture != 0)
glDeleteTextures(1, &mLayerTempTexture);
if (mFBOTexture != 0)
glDeleteTextures(1, &mFBOTexture);
if (mOutputTexture != 0)
glDeleteTextures(1, &mOutputTexture);
if (mOutputFrameBuf != 0)
glDeleteFramebuffers(1, &mOutputFrameBuf);
if (mUnpinnedTextureBuffer != 0)
glDeleteBuffers(1, &mUnpinnedTextureBuffer);
destroyTemporalHistoryResources();
destroyLayerPrograms();
destroyDecodeShaderProgram();
if (mOscServer) if (mOscServer)
mOscServer->Stop(); mOscServer->Stop();
if (mControlServer) if (mControlServer)
@@ -418,7 +364,7 @@ bool OpenGLComposite::InitDeckLink()
} }
if (mInputFrameWidth != mOutputFrameWidth || mInputFrameHeight != mOutputFrameHeight) if (mInputFrameWidth != mOutputFrameWidth || mInputFrameHeight != mOutputFrameHeight)
{ {
mFastTransferExtensionAvailable = false; mRenderer->mFastTransferExtensionAvailable = false;
OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n"); OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n");
} }
@@ -451,10 +397,10 @@ bool OpenGLComposite::InitDeckLink()
else else
resizeWindow(mOutputFrameWidth / 2, mOutputFrameHeight / 2); resizeWindow(mOutputFrameWidth / 2, mOutputFrameHeight / 2);
if (mFastTransferExtensionAvailable) if (mRenderer->mFastTransferExtensionAvailable)
{ {
// Initialize fast video frame transfers // Initialize fast video frame transfers
if (! VideoFrameTransfer::initialize(mInputFrameWidth, mInputFrameHeight, mCaptureTexture, mOutputTexture)) if (! VideoFrameTransfer::initialize(mInputFrameWidth, mInputFrameHeight, mRenderer->mCaptureTexture, mRenderer->mOutputTexture))
{ {
MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK); MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK);
goto error; goto error;
@@ -661,44 +607,7 @@ void OpenGLComposite::paintGL()
return; return;
} }
// The DeckLink API provides IDeckLinkGLScreenPreviewHelper as a convenient way to view the playout video frames mRenderer->PresentToWindow(hGLDC, mOutputFrameWidth, mOutputFrameHeight);
// in a window. However, it performs a copy from host memory to the GPU which is wasteful in this case since
// we already have the rendered frame to be played out sitting in the GPU in the mIdFrameBuf frame buffer.
// Copy the off-screen frame buffer to the on-screen frame buffer while preserving the
// incoming video aspect ratio. Any extra window area is cleared to black.
int destWidth = mViewWidth;
int destHeight = mViewHeight;
int destX = 0;
int destY = 0;
if (mOutputFrameWidth > 0 && mOutputFrameHeight > 0 && mViewWidth > 0 && mViewHeight > 0)
{
const double frameAspect = static_cast<double>(mOutputFrameWidth) / static_cast<double>(mOutputFrameHeight);
const double viewAspect = static_cast<double>(mViewWidth) / static_cast<double>(mViewHeight);
if (viewAspect > frameAspect)
{
destHeight = mViewHeight;
destWidth = static_cast<int>(destHeight * frameAspect + 0.5);
destX = (mViewWidth - destWidth) / 2;
}
else
{
destWidth = mViewWidth;
destHeight = static_cast<int>(destWidth / frameAspect + 0.5);
destY = (mViewHeight - destHeight) / 2;
}
}
glBindFramebuffer(GL_READ_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
glViewport(0, 0, mViewWidth, mViewHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glBlitFramebuffer(0, 0, mOutputFrameWidth, mOutputFrameHeight, destX, destY, destX + destWidth, destY + destHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
SwapBuffers(hGLDC);
ValidateRect(hGLWnd, NULL); ValidateRect(hGLWnd, NULL);
LeaveCriticalSection(&pMutex); LeaveCriticalSection(&pMutex);
} }
@@ -707,8 +616,7 @@ void OpenGLComposite::resizeGL(WORD width, WORD height)
{ {
// We don't set the project or model matrices here since the window data is copied directly from // We don't set the project or model matrices here since the window data is copied directly from
// an off-screen FBO in paintGL(). Just save the width and height for use in paintGL(). // an off-screen FBO in paintGL(). Just save the width and height for use in paintGL().
mViewWidth = width; mRenderer->ResizeView(width, height);
mViewHeight = height;
} }
void OpenGLComposite::resizeWindow(int width, int height) void OpenGLComposite::resizeWindow(int width, int height)
@@ -753,132 +661,13 @@ bool OpenGLComposite::InitOpenGLState()
} }
resetTemporalHistoryState(); resetTemporalHistoryState();
glClearColor( 0.0f, 0.0f, 0.0f, 0.5f ); // Black background std::string rendererError;
glDisable(GL_DEPTH_TEST); if (!mRenderer->InitializeResources(mInputFrameWidth, mInputFrameHeight, mOutputFrameWidth, mOutputFrameHeight, rendererError))
if (! mFastTransferExtensionAvailable)
{ {
glGenBuffers(1, &mUnpinnedTextureBuffer); MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK);
}
// Setup the texture which will hold the captured video frame pixels
glGenTextures(1, &mCaptureTexture);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
// Parameters to control how texels are sampled from the texture
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// Create texture with empty data, we will update it using glTexSubImage2D each frame.
// The captured video is YCbCr 4:2:2 packed into a UYVY macropixel. OpenGL has no YCbCr format
// so treat it as RGBA 4:4:4:4 by halving the width and using GL_RGBA internal format.
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth / 2, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mDecodedTexture);
glBindTexture(GL_TEXTURE_2D, mDecodedTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mLayerTempTexture);
glBindTexture(GL_TEXTURE_2D, mLayerTempTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
// Create Frame Buffer Object (FBO) to perform off-screen rendering of scene.
// This allows the render to be done on a framebuffer with width and height exactly matching the video format.
glGenFramebuffers(1, &mDecodeFrameBuf);
glGenFramebuffers(1, &mLayerTempFrameBuf);
glGenFramebuffers(1, &mIdFrameBuf);
glGenFramebuffers(1, &mOutputFrameBuf);
glGenRenderbuffers(1, &mIdColorBuf);
glGenRenderbuffers(1, &mIdDepthBuf);
glGenVertexArrays(1, &mFullscreenVAO);
glGenBuffers(1, &mGlobalParamsUBO);
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mDecodedTexture, 0);
GLenum glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize decode framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false; return false;
} }
glBindFramebuffer(GL_FRAMEBUFFER, mLayerTempFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mLayerTempTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize layer framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
// Texture for FBO
glGenTextures(1, &mFBOTexture);
glBindTexture(GL_TEXTURE_2D, mFBOTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
// Attach a depth buffer
glBindRenderbuffer(GL_RENDERBUFFER, mIdDepthBuf);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, mInputFrameWidth, mInputFrameHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER, mIdDepthBuf);
// Attach the texture which stores the playback image
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mFBOTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glGenTextures(1, &mOutputTexture);
glBindTexture(GL_TEXTURE_2D, mOutputTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mOutputFrameWidth, mOutputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mOutputTexture, 0);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize output framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glBindTexture(GL_TEXTURE_2D, 0);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindVertexArray(mFullscreenVAO);
glBindVertexArray(0);
glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO);
glBufferData(GL_UNIFORM_BUFFER, 1024, NULL, GL_DYNAMIC_DRAW);
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO);
glBindBuffer(GL_UNIFORM_BUFFER, 0);
broadcastRuntimeState(); broadcastRuntimeState();
return true; return true;
} }
@@ -913,10 +702,10 @@ void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bo
wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread
if (mFastTransferExtensionAvailable) if (mRenderer->mFastTransferExtensionAvailable)
{ {
CComQIPtr<PinnedMemoryAllocator, &IID_PinnedMemoryAllocator> allocator(inputFrameBuffer); CComQIPtr<PinnedMemoryAllocator, &IID_PinnedMemoryAllocator> allocator(inputFrameBuffer);
if (!allocator || !allocator->transferFrame(videoPixels, mCaptureTexture)) if (!allocator || !allocator->transferFrame(videoPixels, mRenderer->mCaptureTexture))
OutputDebugStringA("Capture: transferFrame() failed\n"); OutputDebugStringA("Capture: transferFrame() failed\n");
allocator->waitForTransferComplete(videoPixels); allocator->waitForTransferComplete(videoPixels);
@@ -924,9 +713,9 @@ void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bo
else else
{ {
// Use a straightforward texture buffer // Use a straightforward texture buffer
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mUnpinnedTextureBuffer); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer->mUnpinnedTextureBuffer);
glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture); glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture);
// NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data // NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mInputFrameWidth / 2, mInputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mInputFrameWidth / 2, mInputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
@@ -959,16 +748,16 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
// Draw the effect output to the off-screen framebuffer. // Draw the effect output to the off-screen framebuffer.
const auto renderStartTime = std::chrono::steady_clock::now(); const auto renderStartTime = std::chrono::steady_clock::now();
if (mFastTransferExtensionAvailable) if (mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU); VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU);
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf);
renderEffect(); renderEffect();
glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mIdFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mOutputFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glFlush(); glFlush();
if (mFastTransferExtensionAvailable) if (mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU); VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU);
const auto renderEndTime = std::chrono::steady_clock::now(); const auto renderEndTime = std::chrono::steady_clock::now();
if (mRuntimeHost) if (mRuntimeHost)
@@ -999,13 +788,13 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
void* pFrame; void* pFrame;
outputVideoFrameBuffer->GetBytes(&pFrame); outputVideoFrameBuffer->GetBytes(&pFrame);
if (mFastTransferExtensionAvailable) if (mRenderer->mFastTransferExtensionAvailable)
{ {
// Finished with mCaptureTexture // Finished with mRenderer->mCaptureTexture
if (!mHasNoInputSource) if (!mHasNoInputSource)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
if (! mPlayoutAllocator->transferFrame(pFrame, mOutputTexture)) if (! mPlayoutAllocator->transferFrame(pFrame, mRenderer->mOutputTexture))
OutputDebugStringA("Playback: transferFrame() failed\n"); OutputDebugStringA("Playback: transferFrame() failed\n");
paintGL(); paintGL();
@@ -1015,7 +804,7 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
} }
else else
{ {
glBindFramebuffer(GL_READ_FRAMEBUFFER, mOutputFrameBuf); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mOutputFrameBuf);
glReadPixels(0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); glReadPixels(0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame);
paintGL(); paintGL();
} }
@@ -1342,7 +1131,7 @@ bool OpenGLComposite::compileLayerPrograms(int errorMessageSize, char* errorMess
} }
destroyLayerPrograms(); destroyLayerPrograms();
mLayerPrograms.swap(newPrograms); mRenderer->mLayerPrograms.swap(newPrograms);
if (mRuntimeHost) if (mRuntimeHost)
{ {
@@ -1393,57 +1182,20 @@ bool OpenGLComposite::compileDecodeShader(int errorMessageSize, char* errorMessa
} }
destroyDecodeShaderProgram(); destroyDecodeShaderProgram();
mDecodeProgram = newProgram.release(); mRenderer->mDecodeProgram = newProgram.release();
mDecodeVertexShader = newVertexShader.release(); mRenderer->mDecodeVertexShader = newVertexShader.release();
mDecodeFragmentShader = newFragmentShader.release(); mRenderer->mDecodeFragmentShader = newFragmentShader.release();
return true; return true;
} }
void OpenGLComposite::destroySingleLayerProgram(LayerProgram& layerProgram) void OpenGLComposite::destroySingleLayerProgram(LayerProgram& layerProgram)
{ {
for (LayerProgram::TextureBinding& textureBinding : layerProgram.textureBindings) mRenderer->DestroySingleLayerProgram(layerProgram);
{
if (textureBinding.texture != 0)
{
glDeleteTextures(1, &textureBinding.texture);
textureBinding.texture = 0;
}
}
layerProgram.textureBindings.clear();
for (LayerProgram::TextBinding& textBinding : layerProgram.textBindings)
{
if (textBinding.texture != 0)
{
glDeleteTextures(1, &textBinding.texture);
textBinding.texture = 0;
}
}
layerProgram.textBindings.clear();
if (layerProgram.program != 0)
{
glDeleteProgram(layerProgram.program);
layerProgram.program = 0;
}
if (layerProgram.fragmentShader != 0)
{
glDeleteShader(layerProgram.fragmentShader);
layerProgram.fragmentShader = 0;
}
if (layerProgram.vertexShader != 0)
{
glDeleteShader(layerProgram.vertexShader);
layerProgram.vertexShader = 0;
}
} }
void OpenGLComposite::destroyLayerPrograms() void OpenGLComposite::destroyLayerPrograms()
{ {
for (LayerProgram& layerProgram : mLayerPrograms) mRenderer->DestroyLayerPrograms();
destroySingleLayerProgram(layerProgram);
mLayerPrograms.clear();
} }
bool OpenGLComposite::loadTextureAsset(const ShaderTextureAsset& textureAsset, GLuint& textureId, std::string& error) bool OpenGLComposite::loadTextureAsset(const ShaderTextureAsset& textureAsset, GLuint& textureId, std::string& error)
@@ -1508,241 +1260,40 @@ void OpenGLComposite::bindLayerTextureAssets(const LayerProgram& layerProgram)
void OpenGLComposite::destroyDecodeShaderProgram() void OpenGLComposite::destroyDecodeShaderProgram()
{ {
if (mDecodeProgram != 0) mRenderer->DestroyDecodeShaderProgram();
{
glDeleteProgram(mDecodeProgram);
mDecodeProgram = 0;
}
if (mDecodeFragmentShader != 0)
{
glDeleteShader(mDecodeFragmentShader);
mDecodeFragmentShader = 0;
}
if (mDecodeVertexShader != 0)
{
glDeleteShader(mDecodeVertexShader);
mDecodeVertexShader = 0;
}
} }
bool OpenGLComposite::validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const bool OpenGLComposite::validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const
{ {
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
unsigned requiredUnits = kSourceHistoryTextureUnitBase; return mRenderer->mTemporalHistory.ValidateTextureUnitBudget(layerStates, historyCap, error);
for (const RuntimeRenderState& state : layerStates)
{
unsigned textTextureCount = 0;
for (const ShaderParameterDefinition& definition : state.parameterDefinitions)
{
if (definition.type == ShaderParameterType::Text)
++textTextureCount;
}
const unsigned totalShaderTextures = static_cast<unsigned>(state.textureAssets.size()) + textTextureCount;
const unsigned layerRequiredUnits = kSourceHistoryTextureUnitBase + (state.isTemporal ? historyCap + historyCap : 0u) + totalShaderTextures;
if (layerRequiredUnits > requiredUnits)
requiredUnits = layerRequiredUnits;
}
GLint maxTextureUnits = 0;
glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits);
const unsigned availableUnits = maxTextureUnits > 0 ? static_cast<unsigned>(maxTextureUnits) : 0u;
if (requiredUnits > availableUnits)
{
std::ostringstream message;
message << "The current history and shader texture asset configuration requires " << requiredUnits
<< " fragment texture units, but only " << maxTextureUnits << " are available.";
error = message.str();
return false;
}
return true;
}
bool OpenGLComposite::createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error)
{
destroyHistoryRing(ring);
ring.effectiveLength = effectiveLength;
ring.historySource = historySource;
if (effectiveLength == 0)
return true;
ring.slots.resize(effectiveLength);
for (HistorySlot& slot : ring.slots)
{
glGenTextures(1, &slot.texture);
glBindTexture(GL_TEXTURE_2D, slot.texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mInputFrameWidth, mInputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glGenFramebuffers(1, &slot.framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, slot.framebuffer);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, slot.texture, 0);
const GLenum framebufferStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (framebufferStatus != GL_FRAMEBUFFER_COMPLETE)
{
error = "Failed to initialize a temporal history framebuffer.";
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
destroyHistoryRing(ring);
return false;
}
}
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
return true;
}
void OpenGLComposite::destroyHistoryRing(HistoryRing& ring)
{
for (HistorySlot& slot : ring.slots)
{
if (slot.framebuffer != 0)
glDeleteFramebuffers(1, &slot.framebuffer);
if (slot.texture != 0)
glDeleteTextures(1, &slot.texture);
slot.framebuffer = 0;
slot.texture = 0;
}
ring.slots.clear();
ring.nextWriteIndex = 0;
ring.filledCount = 0;
ring.effectiveLength = 0;
ring.historySource = TemporalHistorySource::None;
}
void OpenGLComposite::destroyTemporalHistoryResources()
{
destroyHistoryRing(mSourceHistoryRing);
for (auto& historyEntry : mPreLayerHistoryByLayerId)
destroyHistoryRing(historyEntry.second);
mPreLayerHistoryByLayerId.clear();
} }
void OpenGLComposite::resetTemporalHistoryState() void OpenGLComposite::resetTemporalHistoryState()
{ {
mSourceHistoryRing.nextWriteIndex = 0; mRenderer->mTemporalHistory.ResetState();
mSourceHistoryRing.filledCount = 0;
for (auto& historyEntry : mPreLayerHistoryByLayerId)
{
historyEntry.second.nextWriteIndex = 0;
historyEntry.second.filledCount = 0;
}
mTemporalHistoryNeedsReset = false;
} }
bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error) bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error)
{ {
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
const bool sourceHistoryNeeded = std::any_of(layerStates.begin(), layerStates.end(), return mRenderer->mTemporalHistory.EnsureResources(layerStates, historyCap, mInputFrameWidth, mInputFrameHeight, error);
[](const RuntimeRenderState& state) { return state.isTemporal && state.effectiveTemporalHistoryLength > 0; });
const unsigned sourceHistoryLength = sourceHistoryNeeded ? historyCap : 0;
if (mSourceHistoryRing.effectiveLength != sourceHistoryLength)
{
if (!createHistoryRing(mSourceHistoryRing, sourceHistoryLength, TemporalHistorySource::Source, error))
return false;
mTemporalHistoryNeedsReset = true;
}
std::set<std::string> requiredPreLayerIds;
for (const RuntimeRenderState& state : layerStates)
{
if (!state.isTemporal || state.temporalHistorySource != TemporalHistorySource::PreLayerInput)
continue;
requiredPreLayerIds.insert(state.layerId);
auto historyIt = mPreLayerHistoryByLayerId.find(state.layerId);
if (historyIt == mPreLayerHistoryByLayerId.end() || historyIt->second.effectiveLength != state.effectiveTemporalHistoryLength)
{
HistoryRing replacement;
if (!createHistoryRing(replacement, state.effectiveTemporalHistoryLength, TemporalHistorySource::PreLayerInput, error))
return false;
mPreLayerHistoryByLayerId[state.layerId] = std::move(replacement);
mTemporalHistoryNeedsReset = true;
}
}
for (auto it = mPreLayerHistoryByLayerId.begin(); it != mPreLayerHistoryByLayerId.end();)
{
if (requiredPreLayerIds.find(it->first) == requiredPreLayerIds.end())
{
destroyHistoryRing(it->second);
it = mPreLayerHistoryByLayerId.erase(it);
mTemporalHistoryNeedsReset = true;
}
else
{
++it;
}
}
if (mTemporalHistoryNeedsReset)
resetTemporalHistoryState();
return true;
}
void OpenGLComposite::pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring)
{
if (ring.effectiveLength == 0 || ring.slots.empty())
return;
HistorySlot& targetSlot = ring.slots[ring.nextWriteIndex];
glBindFramebuffer(GL_READ_FRAMEBUFFER, sourceFramebuffer);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, targetSlot.framebuffer);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
ring.nextWriteIndex = (ring.nextWriteIndex + 1) % ring.slots.size();
ring.filledCount = std::min<std::size_t>(ring.filledCount + 1, ring.slots.size());
}
GLuint OpenGLComposite::resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const
{
if (ring.filledCount == 0 || ring.slots.empty())
return fallbackTexture;
const std::size_t clampedOffset = std::min<std::size_t>(framesAgo, ring.filledCount - 1);
const std::size_t newestIndex = (ring.nextWriteIndex + ring.slots.size() - 1) % ring.slots.size();
const std::size_t slotIndex = (newestIndex + ring.slots.size() - clampedOffset) % ring.slots.size();
return ring.slots[slotIndex].texture != 0 ? ring.slots[slotIndex].texture : fallbackTexture;
} }
unsigned OpenGLComposite::sourceHistoryAvailableCount() const unsigned OpenGLComposite::sourceHistoryAvailableCount() const
{ {
return static_cast<unsigned>(mSourceHistoryRing.filledCount); return mRenderer->mTemporalHistory.SourceAvailableCount();
} }
unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const
{ {
auto it = mPreLayerHistoryByLayerId.find(layerId); return mRenderer->mTemporalHistory.AvailableCountForLayer(layerId);
if (it == mPreLayerHistoryByLayerId.end())
return 0;
return static_cast<unsigned>(it->second.filledCount);
} }
void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture) void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture)
{ {
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
for (unsigned index = 0; index < historyCap; ++index) mRenderer->mTemporalHistory.BindSamplers(state, currentSourceTexture, historyCap);
{
glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index);
glBindTexture(GL_TEXTURE_2D, resolveHistoryTexture(mSourceHistoryRing, currentSourceTexture, index));
}
const GLuint temporalBase = kSourceHistoryTextureUnitBase + historyCap;
const HistoryRing* temporalRing = nullptr;
auto it = mPreLayerHistoryByLayerId.find(state.layerId);
if (it != mPreLayerHistoryByLayerId.end())
temporalRing = &it->second;
for (unsigned index = 0; index < historyCap; ++index)
{
glActiveTexture(GL_TEXTURE0 + temporalBase + index);
glBindTexture(GL_TEXTURE_2D, temporalRing ? resolveHistoryTexture(*temporalRing, currentSourceTexture, index) : currentSourceTexture);
}
glActiveTexture(GL_TEXTURE0);
} }
void OpenGLComposite::renderEffect() void OpenGLComposite::renderEffect()
@@ -1750,9 +1301,9 @@ void OpenGLComposite::renderEffect()
PollRuntimeChanges(); PollRuntimeChanges();
const bool hasInputSource = !mHasNoInputSource; const bool hasInputSource = !mHasNoInputSource;
if (hasInputSource && mFastTransferExtensionAvailable) if (hasInputSource && mRenderer->mFastTransferExtensionAvailable)
{ {
// Signal that we're about to draw using mCaptureTexture onto mFBOTexture. // Signal that we're about to draw using mRenderer->mCaptureTexture onto mRenderer->mFBOTexture.
VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU); VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU);
} }
@@ -1764,43 +1315,39 @@ void OpenGLComposite::renderEffect()
} }
else else
{ {
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glViewport(0, 0, mInputFrameWidth, mInputFrameHeight); glViewport(0, 0, mInputFrameWidth, mInputFrameHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT); glClear(GL_COLOR_BUFFER_BIT);
} }
const std::vector<RuntimeRenderState> layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mInputFrameWidth, mInputFrameHeight) : std::vector<RuntimeRenderState>(); const std::vector<RuntimeRenderState> layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mInputFrameWidth, mInputFrameHeight) : std::vector<RuntimeRenderState>();
if (layerStates.empty() || mLayerPrograms.empty()) if (layerStates.empty() || mRenderer->mLayerPrograms.empty())
{ {
glBindFramebuffer(GL_READ_FRAMEBUFFER, mDecodeFrameBuf); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mIdFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mIdFrameBuf);
glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf);
} }
else else
{ {
GLuint sourceTexture = mDecodedTexture; GLuint sourceTexture = mRenderer->mDecodedTexture;
GLuint sourceFrameBuffer = mDecodeFrameBuf; GLuint sourceFrameBuffer = mRenderer->mDecodeFrameBuf;
for (std::size_t index = 0; index < layerStates.size() && index < mLayerPrograms.size(); ++index) for (std::size_t index = 0; index < layerStates.size() && index < mRenderer->mLayerPrograms.size(); ++index)
{ {
const std::size_t remaining = layerStates.size() - index; const std::size_t remaining = layerStates.size() - index;
const bool writeToMain = (remaining % 2) == 1; const bool writeToMain = (remaining % 2) == 1;
renderShaderProgram(sourceTexture, writeToMain ? mIdFrameBuf : mLayerTempFrameBuf, mLayerPrograms[index], layerStates[index]); renderShaderProgram(sourceTexture, writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf, mRenderer->mLayerPrograms[index], layerStates[index]);
if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput) if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput)
{ mRenderer->mTemporalHistory.PushPreLayerFramebuffer(layerStates[index].layerId, sourceFrameBuffer, mInputFrameWidth, mInputFrameHeight);
auto historyIt = mPreLayerHistoryByLayerId.find(layerStates[index].layerId); sourceTexture = writeToMain ? mRenderer->mFBOTexture : mRenderer->mLayerTempTexture;
if (historyIt != mPreLayerHistoryByLayerId.end()) sourceFrameBuffer = writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf;
pushFramebufferToHistoryRing(sourceFrameBuffer, historyIt->second);
}
sourceTexture = writeToMain ? mFBOTexture : mLayerTempTexture;
sourceFrameBuffer = writeToMain ? mIdFrameBuf : mLayerTempFrameBuf;
} }
} }
pushFramebufferToHistoryRing(mDecodeFrameBuf, mSourceHistoryRing); mRenderer->mTemporalHistory.PushSourceFramebuffer(mRenderer->mDecodeFrameBuf, mInputFrameWidth, mInputFrameHeight);
if (hasInputSource && mFastTransferExtensionAvailable) if (hasInputSource && mRenderer->mFastTransferExtensionAvailable)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
} }
@@ -1820,7 +1367,7 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
glBindTexture(GL_TEXTURE_2D, sourceTexture); glBindTexture(GL_TEXTURE_2D, sourceTexture);
bindHistorySamplers(state, sourceTexture); bindHistorySamplers(state, sourceTexture);
bindLayerTextureAssets(layerProgram); bindLayerTextureAssets(layerProgram);
glBindVertexArray(mFullscreenVAO); glBindVertexArray(mRenderer->mFullscreenVAO);
glUseProgram(layerProgram.program); glUseProgram(layerProgram.program);
updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId)); updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId));
glDrawArrays(GL_TRIANGLES, 0, 3); glDrawArrays(GL_TRIANGLES, 0, 3);
@@ -1847,16 +1394,16 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
void OpenGLComposite::renderDecodePass() void OpenGLComposite::renderDecodePass()
{ {
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf);
glViewport(0, 0, mInputFrameWidth, mInputFrameHeight); glViewport(0, 0, mInputFrameWidth, mInputFrameHeight);
glClear(GL_COLOR_BUFFER_BIT); glClear(GL_COLOR_BUFFER_BIT);
glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit); glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture); glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture);
glBindVertexArray(mFullscreenVAO); glBindVertexArray(mRenderer->mFullscreenVAO);
glUseProgram(mDecodeProgram); glUseProgram(mRenderer->mDecodeProgram);
const GLint packedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uPackedVideoResolution"); const GLint packedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uPackedVideoResolution");
const GLint decodedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uDecodedVideoResolution"); const GLint decodedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uDecodedVideoResolution");
if (packedResolutionLocation >= 0) if (packedResolutionLocation >= 0)
glUniform2f(packedResolutionLocation, static_cast<float>(mInputFrameWidth / 2), static_cast<float>(mInputFrameHeight)); glUniform2f(packedResolutionLocation, static_cast<float>(mInputFrameWidth / 2), static_cast<float>(mInputFrameHeight));
if (decodedResolutionLocation >= 0) if (decodedResolutionLocation >= 0)
@@ -1979,17 +1526,17 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state,
buffer.resize(AlignStd140(buffer.size(), 16), 0); buffer.resize(AlignStd140(buffer.size(), 16), 0);
glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO); glBindBuffer(GL_UNIFORM_BUFFER, mRenderer->mGlobalParamsUBO);
if (mGlobalParamsUBOSize != static_cast<GLsizeiptr>(buffer.size())) if (mRenderer->mGlobalParamsUBOSize != static_cast<GLsizeiptr>(buffer.size()))
{ {
glBufferData(GL_UNIFORM_BUFFER, static_cast<GLsizeiptr>(buffer.size()), buffer.data(), GL_DYNAMIC_DRAW); glBufferData(GL_UNIFORM_BUFFER, static_cast<GLsizeiptr>(buffer.size()), buffer.data(), GL_DYNAMIC_DRAW);
mGlobalParamsUBOSize = static_cast<GLsizeiptr>(buffer.size()); mRenderer->mGlobalParamsUBOSize = static_cast<GLsizeiptr>(buffer.size());
} }
else else
{ {
glBufferSubData(GL_UNIFORM_BUFFER, 0, static_cast<GLsizeiptr>(buffer.size()), buffer.data()); glBufferSubData(GL_UNIFORM_BUFFER, 0, static_cast<GLsizeiptr>(buffer.size()), buffer.data());
} }
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO); glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mRenderer->mGlobalParamsUBO);
glBindBuffer(GL_UNIFORM_BUFFER, 0); glBindBuffer(GL_UNIFORM_BUFFER, 0);
return true; return true;
@@ -1997,12 +1544,13 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state,
bool OpenGLComposite::CheckOpenGLExtensions() bool OpenGLComposite::CheckOpenGLExtensions()
{ {
mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable(); mRenderer->mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable();
if (!mFastTransferExtensionAvailable) if (!mRenderer->mFastTransferExtensionAvailable)
OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n"); OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n");
return true; return true;
} }
//////////////////////////////////////////// ////////////////////////////////////////////

80
apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.h
View File

@@ -53,6 +53,7 @@
#include "DeckLinkAPI_h.h" #include "DeckLinkAPI_h.h"
#include "GLExtensions.h" #include "GLExtensions.h"
#include "OpenGLRenderer.h"
#include "RuntimeHost.h" #include "RuntimeHost.h"
#include <atomic> #include <atomic>
@@ -101,6 +102,7 @@ public:
private: private:
void resizeWindow(int width, int height); void resizeWindow(int width, int height);
bool CheckOpenGLExtensions(); bool CheckOpenGLExtensions();
using LayerProgram = OpenGLRenderer::LayerProgram;
CaptureDelegate* mCaptureDelegate; CaptureDelegate* mCaptureDelegate;
PlayoutDelegate* mPlayoutDelegate; PlayoutDelegate* mPlayoutDelegate;
@@ -132,82 +134,11 @@ private:
bool mDeckLinkExternalKeyingActive; bool mDeckLinkExternalKeyingActive;
std::string mDeckLinkStatusMessage; std::string mDeckLinkStatusMessage;
// OpenGL data std::unique_ptr<OpenGLRenderer> mRenderer;
bool mFastTransferExtensionAvailable;
GLuint mCaptureTexture;
GLuint mDecodedTexture;
GLuint mLayerTempTexture;
GLuint mFBOTexture;
GLuint mOutputTexture;
GLuint mUnpinnedTextureBuffer;
GLuint mDecodeFrameBuf;
GLuint mLayerTempFrameBuf;
GLuint mIdFrameBuf;
GLuint mOutputFrameBuf;
GLuint mIdColorBuf;
GLuint mIdDepthBuf;
GLuint mFullscreenVAO;
GLuint mGlobalParamsUBO;
GLuint mDecodeProgram;
GLuint mDecodeVertexShader;
GLuint mDecodeFragmentShader;
GLsizeiptr mGlobalParamsUBOSize;
int mViewWidth;
int mViewHeight;
std::unique_ptr<RuntimeHost> mRuntimeHost; std::unique_ptr<RuntimeHost> mRuntimeHost;
std::unique_ptr<ControlServer> mControlServer; std::unique_ptr<ControlServer> mControlServer;
std::unique_ptr<OscServer> mOscServer; std::unique_ptr<OscServer> mOscServer;
struct LayerProgram
{
struct TextureBinding
{
std::string samplerName;
std::filesystem::path sourcePath;
GLuint texture = 0;
};
struct TextBinding
{
std::string parameterId;
std::string samplerName;
std::string fontId;
GLuint texture = 0;
std::string renderedText;
unsigned renderedWidth = 0;
unsigned renderedHeight = 0;
};
std::string layerId;
std::string shaderId;
GLuint shaderTextureBase = 0;
GLuint program = 0;
GLuint vertexShader = 0;
GLuint fragmentShader = 0;
std::vector<TextureBinding> textureBindings;
std::vector<TextBinding> textBindings;
};
std::vector<LayerProgram> mLayerPrograms;
struct HistorySlot
{
GLuint texture = 0;
GLuint framebuffer = 0;
};
struct HistoryRing
{
std::vector<HistorySlot> slots;
std::size_t nextWriteIndex = 0;
std::size_t filledCount = 0;
unsigned effectiveLength = 0;
TemporalHistorySource historySource = TemporalHistorySource::None;
};
HistoryRing mSourceHistoryRing;
std::map<std::string, HistoryRing> mPreLayerHistoryByLayerId;
bool mTemporalHistoryNeedsReset;
bool InitOpenGLState(); bool InitOpenGLState();
bool compileLayerPrograms(int errorMessageSize, char* errorMessage); bool compileLayerPrograms(int errorMessageSize, char* errorMessage);
bool compileSingleLayerProgram(const RuntimeRenderState& state, LayerProgram& layerProgram, int errorMessageSize, char* errorMessage); bool compileSingleLayerProgram(const RuntimeRenderState& state, LayerProgram& layerProgram, int errorMessageSize, char* errorMessage);
@@ -226,13 +157,8 @@ private:
bool updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength); bool updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength);
bool validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const; bool validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const;
bool ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error); bool ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error);
bool createHistoryRing(HistoryRing& ring, unsigned effectiveLength, TemporalHistorySource historySource, std::string& error);
void destroyHistoryRing(HistoryRing& ring);
void destroyTemporalHistoryResources();
void resetTemporalHistoryState(); void resetTemporalHistoryState();
void pushFramebufferToHistoryRing(GLuint sourceFramebuffer, HistoryRing& ring);
void bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture); void bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture);
GLuint resolveHistoryTexture(const HistoryRing& ring, GLuint fallbackTexture, std::size_t framesAgo) const;
unsigned sourceHistoryAvailableCount() const; unsigned sourceHistoryAvailableCount() const;
unsigned temporalHistoryAvailableCountForLayer(const std::string& layerId) const; unsigned temporalHistoryAvailableCountForLayer(const std::string& layerId) const;
}; };

266
apps/LoopThroughWithOpenGLCompositing/gl/OpenGLRenderer.cpp Normal file
View File

@@ -0,0 +1,266 @@
#include "OpenGLRenderer.h"
#include "GlRenderConstants.h"
namespace
{
void ConfigureFrameTexture(unsigned width, unsigned height)
{
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
}
}
bool OpenGLRenderer::InitializeResources(unsigned inputFrameWidth, unsigned inputFrameHeight, unsigned outputFrameWidth, unsigned outputFrameHeight, std::string& error)
{
glClearColor(0.0f, 0.0f, 0.0f, 0.5f);
glDisable(GL_DEPTH_TEST);
if (!mFastTransferExtensionAvailable)
glGenBuffers(1, &mUnpinnedTextureBuffer);
glGenTextures(1, &mCaptureTexture);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
ConfigureFrameTexture(inputFrameWidth / 2, inputFrameHeight);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mDecodedTexture);
glBindTexture(GL_TEXTURE_2D, mDecodedTexture);
ConfigureFrameTexture(inputFrameWidth, inputFrameHeight);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mLayerTempTexture);
glBindTexture(GL_TEXTURE_2D, mLayerTempTexture);
ConfigureFrameTexture(inputFrameWidth, inputFrameHeight);
glBindTexture(GL_TEXTURE_2D, 0);
glGenFramebuffers(1, &mDecodeFrameBuf);
glGenFramebuffers(1, &mLayerTempFrameBuf);
glGenFramebuffers(1, &mIdFrameBuf);
glGenFramebuffers(1, &mOutputFrameBuf);
glGenRenderbuffers(1, &mIdColorBuf);
glGenRenderbuffers(1, &mIdDepthBuf);
glGenVertexArrays(1, &mFullscreenVAO);
glGenBuffers(1, &mGlobalParamsUBO);
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mDecodedTexture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
error = "Cannot initialize decode framebuffer.";
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mLayerTempFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mLayerTempTexture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
error = "Cannot initialize layer framebuffer.";
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
glGenTextures(1, &mFBOTexture);
glBindTexture(GL_TEXTURE_2D, mFBOTexture);
ConfigureFrameTexture(inputFrameWidth, inputFrameHeight);
glBindRenderbuffer(GL_RENDERBUFFER, mIdDepthBuf);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, inputFrameWidth, inputFrameHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT_EXT, GL_RENDERBUFFER, mIdDepthBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mFBOTexture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
error = "Cannot initialize framebuffer.";
return false;
}
glGenTextures(1, &mOutputTexture);
glBindTexture(GL_TEXTURE_2D, mOutputTexture);
ConfigureFrameTexture(outputFrameWidth, outputFrameHeight);
glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mOutputTexture, 0);
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{
error = "Cannot initialize output framebuffer.";
return false;
}
glBindTexture(GL_TEXTURE_2D, 0);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindVertexArray(mFullscreenVAO);
glBindVertexArray(0);
glBindBuffer(GL_UNIFORM_BUFFER, mGlobalParamsUBO);
glBufferData(GL_UNIFORM_BUFFER, 1024, NULL, GL_DYNAMIC_DRAW);
glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mGlobalParamsUBO);
glBindBuffer(GL_UNIFORM_BUFFER, 0);
return true;
}
void OpenGLRenderer::ResizeView(int width, int height)
{
mViewWidth = width;
mViewHeight = height;
}
void OpenGLRenderer::PresentToWindow(HDC hdc, unsigned outputFrameWidth, unsigned outputFrameHeight)
{
int destWidth = mViewWidth;
int destHeight = mViewHeight;
int destX = 0;
int destY = 0;
if (outputFrameWidth > 0 && outputFrameHeight > 0 && mViewWidth > 0 && mViewHeight > 0)
{
const double frameAspect = static_cast<double>(outputFrameWidth) / static_cast<double>(outputFrameHeight);
const double viewAspect = static_cast<double>(mViewWidth) / static_cast<double>(mViewHeight);
if (viewAspect > frameAspect)
{
destHeight = mViewHeight;
destWidth = static_cast<int>(destHeight * frameAspect + 0.5);
destX = (mViewWidth - destWidth) / 2;
}
else
{
destWidth = mViewWidth;
destHeight = static_cast<int>(destWidth / frameAspect + 0.5);
destY = (mViewHeight - destHeight) / 2;
}
}
glBindFramebuffer(GL_READ_FRAMEBUFFER, mOutputFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
glViewport(0, 0, mViewWidth, mViewHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glBlitFramebuffer(0, 0, outputFrameWidth, outputFrameHeight, destX, destY, destX + destWidth, destY + destHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
SwapBuffers(hdc);
}
void OpenGLRenderer::DestroyResources()
{
if (mFullscreenVAO != 0)
glDeleteVertexArrays(1, &mFullscreenVAO);
if (mGlobalParamsUBO != 0)
glDeleteBuffers(1, &mGlobalParamsUBO);
if (mDecodeFrameBuf != 0)
glDeleteFramebuffers(1, &mDecodeFrameBuf);
if (mLayerTempFrameBuf != 0)
glDeleteFramebuffers(1, &mLayerTempFrameBuf);
if (mIdFrameBuf != 0)
glDeleteFramebuffers(1, &mIdFrameBuf);
if (mOutputFrameBuf != 0)
glDeleteFramebuffers(1, &mOutputFrameBuf);
if (mIdColorBuf != 0)
glDeleteRenderbuffers(1, &mIdColorBuf);
if (mIdDepthBuf != 0)
glDeleteRenderbuffers(1, &mIdDepthBuf);
if (mCaptureTexture != 0)
glDeleteTextures(1, &mCaptureTexture);
if (mDecodedTexture != 0)
glDeleteTextures(1, &mDecodedTexture);
if (mLayerTempTexture != 0)
glDeleteTextures(1, &mLayerTempTexture);
if (mFBOTexture != 0)
glDeleteTextures(1, &mFBOTexture);
if (mOutputTexture != 0)
glDeleteTextures(1, &mOutputTexture);
if (mUnpinnedTextureBuffer != 0)
glDeleteBuffers(1, &mUnpinnedTextureBuffer);
mFullscreenVAO = 0;
mGlobalParamsUBO = 0;
mDecodeFrameBuf = 0;
mLayerTempFrameBuf = 0;
mIdFrameBuf = 0;
mOutputFrameBuf = 0;
mIdColorBuf = 0;
mIdDepthBuf = 0;
mCaptureTexture = 0;
mDecodedTexture = 0;
mLayerTempTexture = 0;
mFBOTexture = 0;
mOutputTexture = 0;
mUnpinnedTextureBuffer = 0;
mGlobalParamsUBOSize = 0;
mTemporalHistory.DestroyResources();
DestroyLayerPrograms();
DestroyDecodeShaderProgram();
}
void OpenGLRenderer::DestroySingleLayerProgram(LayerProgram& layerProgram)
{
for (LayerProgram::TextureBinding& binding : layerProgram.textureBindings)
{
if (binding.texture != 0)
{
glDeleteTextures(1, &binding.texture);
binding.texture = 0;
}
}
layerProgram.textureBindings.clear();
for (LayerProgram::TextBinding& binding : layerProgram.textBindings)
{
if (binding.texture != 0)
{
glDeleteTextures(1, &binding.texture);
binding.texture = 0;
}
}
layerProgram.textBindings.clear();
if (layerProgram.program != 0)
{
glDeleteProgram(layerProgram.program);
layerProgram.program = 0;
}
if (layerProgram.fragmentShader != 0)
{
glDeleteShader(layerProgram.fragmentShader);
layerProgram.fragmentShader = 0;
}
if (layerProgram.vertexShader != 0)
{
glDeleteShader(layerProgram.vertexShader);
layerProgram.vertexShader = 0;
}
}
void OpenGLRenderer::DestroyLayerPrograms()
{
for (LayerProgram& layerProgram : mLayerPrograms)
DestroySingleLayerProgram(layerProgram);
mLayerPrograms.clear();
}
void OpenGLRenderer::DestroyDecodeShaderProgram()
{
if (mDecodeProgram != 0)
{
glDeleteProgram(mDecodeProgram);
mDecodeProgram = 0;
}
if (mDecodeFragmentShader != 0)
{
glDeleteShader(mDecodeFragmentShader);
mDecodeFragmentShader = 0;
}
if (mDecodeVertexShader != 0)
{
glDeleteShader(mDecodeVertexShader);
mDecodeVertexShader = 0;
}
}

78
apps/LoopThroughWithOpenGLCompositing/gl/OpenGLRenderer.h Normal file
View File

@@ -0,0 +1,78 @@
#pragma once
#include "GLExtensions.h"
#include "ShaderTypes.h"
#include "TemporalHistoryBuffers.h"
#include <windows.h>
#include <filesystem>
#include <gl/gl.h>
#include <string>
#include <vector>
class OpenGLRenderer
{
public:
struct LayerProgram
{
struct TextureBinding
{
std::string samplerName;
std::filesystem::path sourcePath;
GLuint texture = 0;
};
struct TextBinding
{
std::string parameterId;
std::string samplerName;
std::string fontId;
GLuint texture = 0;
std::string renderedText;
unsigned renderedWidth = 0;
unsigned renderedHeight = 0;
};
std::string layerId;
std::string shaderId;
GLuint shaderTextureBase = 0;
GLuint program = 0;
GLuint vertexShader = 0;
GLuint fragmentShader = 0;
std::vector<TextureBinding> textureBindings;
std::vector<TextBinding> textBindings;
};
bool mFastTransferExtensionAvailable = false;
GLuint mCaptureTexture = 0;
GLuint mDecodedTexture = 0;
GLuint mLayerTempTexture = 0;
GLuint mFBOTexture = 0;
GLuint mOutputTexture = 0;
GLuint mUnpinnedTextureBuffer = 0;
GLuint mDecodeFrameBuf = 0;
GLuint mLayerTempFrameBuf = 0;
GLuint mIdFrameBuf = 0;
GLuint mOutputFrameBuf = 0;
GLuint mIdColorBuf = 0;
GLuint mIdDepthBuf = 0;
GLuint mFullscreenVAO = 0;
GLuint mGlobalParamsUBO = 0;
GLuint mDecodeProgram = 0;
GLuint mDecodeVertexShader = 0;
GLuint mDecodeFragmentShader = 0;
GLsizeiptr mGlobalParamsUBOSize = 0;
int mViewWidth = 0;
int mViewHeight = 0;
std::vector<LayerProgram> mLayerPrograms;
TemporalHistoryBuffers mTemporalHistory;
bool InitializeResources(unsigned inputFrameWidth, unsigned inputFrameHeight, unsigned outputFrameWidth, unsigned outputFrameHeight, std::string& error);
void ResizeView(int width, int height);
void PresentToWindow(HDC hdc, unsigned outputFrameWidth, unsigned outputFrameHeight);
void DestroyResources();
void DestroySingleLayerProgram(LayerProgram& layerProgram);
void DestroyLayerPrograms();
void DestroyDecodeShaderProgram();
};

237
apps/LoopThroughWithOpenGLCompositing/gl/TemporalHistoryBuffers.cpp Normal file
View File

@@ -0,0 +1,237 @@
#include "TemporalHistoryBuffers.h"
#include "GlRenderConstants.h"
#include "ShaderTypes.h"
#include <algorithm>
#include <sstream>
#include <set>
bool TemporalHistoryBuffers::ValidateTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, unsigned historyCap, std::string& error) const
{
unsigned requiredUnits = kSourceHistoryTextureUnitBase;
for (const RuntimeRenderState& state : layerStates)
{
unsigned textTextureCount = 0;
for (const ShaderParameterDefinition& definition : state.parameterDefinitions)
{
if (definition.type == ShaderParameterType::Text)
++textTextureCount;
}
const unsigned totalShaderTextures = static_cast<unsigned>(state.textureAssets.size()) + textTextureCount;
const unsigned layerRequiredUnits = kSourceHistoryTextureUnitBase + (state.isTemporal ? historyCap + historyCap : 0u) + totalShaderTextures;
if (layerRequiredUnits > requiredUnits)
requiredUnits = layerRequiredUnits;
}
GLint maxTextureUnits = 0;
glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits);
const unsigned availableUnits = maxTextureUnits > 0 ? static_cast<unsigned>(maxTextureUnits) : 0u;
if (requiredUnits > availableUnits)
{
std::ostringstream message;
message << "The current history and shader texture asset configuration requires " << requiredUnits
<< " fragment texture units, but only " << maxTextureUnits << " are available.";
error = message.str();
return false;
}
return true;
}
bool TemporalHistoryBuffers::EnsureResources(const std::vector<RuntimeRenderState>& layerStates, unsigned historyCap, unsigned frameWidth, unsigned frameHeight, std::string& error)
{
const bool sourceHistoryNeeded = std::any_of(layerStates.begin(), layerStates.end(),
[](const RuntimeRenderState& state) { return state.isTemporal && state.effectiveTemporalHistoryLength > 0; });
const unsigned sourceHistoryLength = sourceHistoryNeeded ? historyCap : 0;
if (sourceHistoryRing.effectiveLength != sourceHistoryLength)
{
if (!CreateRing(sourceHistoryRing, sourceHistoryLength, TemporalHistorySource::Source, frameWidth, frameHeight, error))
return false;
mNeedsReset = true;
}
std::set<std::string> requiredPreLayerIds;
for (const RuntimeRenderState& state : layerStates)
{
if (!state.isTemporal || state.temporalHistorySource != TemporalHistorySource::PreLayerInput)
continue;
requiredPreLayerIds.insert(state.layerId);
auto historyIt = preLayerHistoryByLayerId.find(state.layerId);
if (historyIt == preLayerHistoryByLayerId.end() || historyIt->second.effectiveLength != state.effectiveTemporalHistoryLength)
{
Ring replacement;
if (!CreateRing(replacement, state.effectiveTemporalHistoryLength, TemporalHistorySource::PreLayerInput, frameWidth, frameHeight, error))
return false;
preLayerHistoryByLayerId[state.layerId] = std::move(replacement);
mNeedsReset = true;
}
}
for (auto it = preLayerHistoryByLayerId.begin(); it != preLayerHistoryByLayerId.end();)
{
if (requiredPreLayerIds.find(it->first) == requiredPreLayerIds.end())
{
DestroyRing(it->second);
it = preLayerHistoryByLayerId.erase(it);
mNeedsReset = true;
}
else
{
++it;
}
}
if (mNeedsReset)
ResetState();
return true;
}
bool TemporalHistoryBuffers::CreateRing(Ring& ring, unsigned effectiveLength, TemporalHistorySource historySource, unsigned frameWidth, unsigned frameHeight, std::string& error)
{
DestroyRing(ring);
ring.effectiveLength = effectiveLength;
ring.historySource = historySource;
if (effectiveLength == 0)
return true;
ring.slots.resize(effectiveLength);
for (Slot& slot : ring.slots)
{
glGenTextures(1, &slot.texture);
glBindTexture(GL_TEXTURE_2D, slot.texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, frameWidth, frameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glGenFramebuffers(1, &slot.framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, slot.framebuffer);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, slot.texture, 0);
const GLenum framebufferStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (framebufferStatus != GL_FRAMEBUFFER_COMPLETE)
{
error = "Failed to initialize a temporal history framebuffer.";
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
DestroyRing(ring);
return false;
}
}
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glBindTexture(GL_TEXTURE_2D, 0);
return true;
}
void TemporalHistoryBuffers::DestroyRing(Ring& ring)
{
for (Slot& slot : ring.slots)
{
if (slot.framebuffer != 0)
glDeleteFramebuffers(1, &slot.framebuffer);
if (slot.texture != 0)
glDeleteTextures(1, &slot.texture);
slot.framebuffer = 0;
slot.texture = 0;
}
ring.slots.clear();
ring.nextWriteIndex = 0;
ring.filledCount = 0;
ring.effectiveLength = 0;
ring.historySource = TemporalHistorySource::None;
}
void TemporalHistoryBuffers::DestroyResources()
{
DestroyRing(sourceHistoryRing);
for (auto& historyEntry : preLayerHistoryByLayerId)
DestroyRing(historyEntry.second);
preLayerHistoryByLayerId.clear();
mNeedsReset = true;
}
void TemporalHistoryBuffers::ResetState()
{
sourceHistoryRing.nextWriteIndex = 0;
sourceHistoryRing.filledCount = 0;
for (auto& historyEntry : preLayerHistoryByLayerId)
{
historyEntry.second.nextWriteIndex = 0;
historyEntry.second.filledCount = 0;
}
mNeedsReset = false;
}
void TemporalHistoryBuffers::PushFramebuffer(GLuint sourceFramebuffer, Ring& ring, unsigned frameWidth, unsigned frameHeight)
{
if (ring.effectiveLength == 0 || ring.slots.empty())
return;
Slot& targetSlot = ring.slots[ring.nextWriteIndex];
glBindFramebuffer(GL_READ_FRAMEBUFFER, sourceFramebuffer);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, targetSlot.framebuffer);
glBlitFramebuffer(0, 0, frameWidth, frameHeight, 0, 0, frameWidth, frameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
ring.nextWriteIndex = (ring.nextWriteIndex + 1) % ring.slots.size();
ring.filledCount = std::min<std::size_t>(ring.filledCount + 1, ring.slots.size());
}
void TemporalHistoryBuffers::PushSourceFramebuffer(GLuint sourceFramebuffer, unsigned frameWidth, unsigned frameHeight)
{
PushFramebuffer(sourceFramebuffer, sourceHistoryRing, frameWidth, frameHeight);
}
void TemporalHistoryBuffers::PushPreLayerFramebuffer(const std::string& layerId, GLuint sourceFramebuffer, unsigned frameWidth, unsigned frameHeight)
{
auto historyIt = preLayerHistoryByLayerId.find(layerId);
if (historyIt != preLayerHistoryByLayerId.end())
PushFramebuffer(sourceFramebuffer, historyIt->second, frameWidth, frameHeight);
}
void TemporalHistoryBuffers::BindSamplers(const RuntimeRenderState& state, GLuint currentSourceTexture, unsigned historyCap)
{
for (unsigned index = 0; index < historyCap; ++index)
{
glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index);
glBindTexture(GL_TEXTURE_2D, ResolveTexture(sourceHistoryRing, currentSourceTexture, index));
}
const GLuint temporalBase = kSourceHistoryTextureUnitBase + historyCap;
const Ring* temporalRing = nullptr;
auto it = preLayerHistoryByLayerId.find(state.layerId);
if (it != preLayerHistoryByLayerId.end())
temporalRing = &it->second;
for (unsigned index = 0; index < historyCap; ++index)
{
glActiveTexture(GL_TEXTURE0 + temporalBase + index);
glBindTexture(GL_TEXTURE_2D, temporalRing ? ResolveTexture(*temporalRing, currentSourceTexture, index) : currentSourceTexture);
}
glActiveTexture(GL_TEXTURE0);
}
GLuint TemporalHistoryBuffers::ResolveTexture(const Ring& ring, GLuint fallbackTexture, std::size_t framesAgo) const
{
if (ring.filledCount == 0 || ring.slots.empty())
return fallbackTexture;
const std::size_t clampedOffset = std::min<std::size_t>(framesAgo, ring.filledCount - 1);
const std::size_t newestIndex = (ring.nextWriteIndex + ring.slots.size() - 1) % ring.slots.size();
const std::size_t slotIndex = (newestIndex + ring.slots.size() - clampedOffset) % ring.slots.size();
return ring.slots[slotIndex].texture != 0 ? ring.slots[slotIndex].texture : fallbackTexture;
}
unsigned TemporalHistoryBuffers::SourceAvailableCount() const
{
return static_cast<unsigned>(sourceHistoryRing.filledCount);
}
unsigned TemporalHistoryBuffers::AvailableCountForLayer(const std::string& layerId) const
{
auto it = preLayerHistoryByLayerId.find(layerId);
if (it == preLayerHistoryByLayerId.end())
return 0;
return static_cast<unsigned>(it->second.filledCount);
}

51
apps/LoopThroughWithOpenGLCompositing/gl/TemporalHistoryBuffers.h Normal file
View File

@@ -0,0 +1,51 @@
#pragma once
#include "GLExtensions.h"
#include "ShaderTypes.h"
#include <windows.h>
#include <gl/gl.h>
#include <map>
#include <string>
#include <vector>
struct RuntimeRenderState;
class TemporalHistoryBuffers
{
public:
struct Slot
{
GLuint texture = 0;
GLuint framebuffer = 0;
};
struct Ring
{
std::vector<Slot> slots;
std::size_t nextWriteIndex = 0;
std::size_t filledCount = 0;
unsigned effectiveLength = 0;
TemporalHistorySource historySource = TemporalHistorySource::None;
};
bool ValidateTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, unsigned historyCap, std::string& error) const;
bool EnsureResources(const std::vector<RuntimeRenderState>& layerStates, unsigned historyCap, unsigned frameWidth, unsigned frameHeight, std::string& error);
bool CreateRing(Ring& ring, unsigned effectiveLength, TemporalHistorySource historySource, unsigned frameWidth, unsigned frameHeight, std::string& error);
void DestroyRing(Ring& ring);
void DestroyResources();
void ResetState();
void PushFramebuffer(GLuint sourceFramebuffer, Ring& ring, unsigned frameWidth, unsigned frameHeight);
void PushSourceFramebuffer(GLuint sourceFramebuffer, unsigned frameWidth, unsigned frameHeight);
void PushPreLayerFramebuffer(const std::string& layerId, GLuint sourceFramebuffer, unsigned frameWidth, unsigned frameHeight);
void BindSamplers(const RuntimeRenderState& state, GLuint currentSourceTexture, unsigned historyCap);
GLuint ResolveTexture(const Ring& ring, GLuint fallbackTexture, std::size_t framesAgo) const;
unsigned SourceAvailableCount() const;
unsigned AvailableCountForLayer(const std::string& layerId) const;
private:
Ring sourceHistoryRing;
std::map<std::string, Ring> preLayerHistoryByLayerId;
bool mNeedsReset = true;
};