aiden/video-shader-toys
1
0
Fork 1
Code Issues Pull Requests Actions Releases 5 Wiki Activity

Working

Browse Source
This commit is contained in:
Aiden
2026-05-02 17:13:53 +10:00
parent 1a4c33b9dc
commit fb1bf01fef
19 changed files with 780 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

233
apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp
View File

@@ -54,7 +54,8 @@ DEFINE_GUID(IID_PinnedMemoryAllocator,
namespace
{
constexpr GLuint kVideoTextureUnit = 1;
constexpr GLuint kDecodedVideoTextureUnit = 1;
constexpr GLuint kPackedVideoTextureUnit = 2;
constexpr GLuint kGlobalParamsBindingPoint = 0;
const char* kDisplayModeName = "1080p59.94";
const char* kVertexShaderSource =
@@ -67,6 +68,31 @@ const char* kVertexShaderSource =
" gl_Position = vec4(positions[gl_VertexID], 0.0, 1.0);\n"
" vTexCoord = texCoords[gl_VertexID];\n"
"}\n";
const char* kDecodeFragmentShaderSource =
"#version 430 core\n"
"layout(binding = 2) uniform sampler2D uPackedVideoInput;\n"
"uniform vec2 uPackedVideoResolution;\n"
"uniform vec2 uDecodedVideoResolution;\n"
"in vec2 vTexCoord;\n"
"layout(location = 0) out vec4 fragColor;\n"
"vec4 rec709YCbCr2rgba(float Y, float Cb, float Cr, float a)\n"
"{\n"
" Y = (Y * 256.0 - 16.0) / 219.0;\n"
" Cb = (Cb * 256.0 - 16.0) / 224.0 - 0.5;\n"
" Cr = (Cr * 256.0 - 16.0) / 224.0 - 0.5;\n"
" return vec4(Y + 1.5748 * Cr, Y - 0.1873 * Cb - 0.4681 * Cr, Y + 1.8556 * Cb, a);\n"
"}\n"
"void main()\n"
"{\n"
" vec2 correctedUv = vec2(vTexCoord.x, 1.0 - vTexCoord.y);\n"
" ivec2 decodedSize = ivec2(max(uDecodedVideoResolution, vec2(1.0, 1.0)));\n"
" ivec2 outputCoord = clamp(ivec2(correctedUv * vec2(decodedSize)), ivec2(0, 0), decodedSize - ivec2(1, 1));\n"
" ivec2 packedSize = ivec2(max(uPackedVideoResolution, vec2(1.0, 1.0)));\n"
" ivec2 packedCoord = ivec2(clamp(outputCoord.x / 2, 0, packedSize.x - 1), clamp(outputCoord.y, 0, packedSize.y - 1));\n"
" vec4 macroPixel = texelFetch(uPackedVideoInput, packedCoord, 0);\n"
" float ySample = (outputCoord.x & 1) != 0 ? macroPixel.a : macroPixel.g;\n"
" fragColor = rec709YCbCr2rgba(ySample, macroPixel.b, macroPixel.r, 1.0);\n"
"}\n";
void CopyErrorMessage(const std::string& message, int errorMessageSize, char* errorMessage)
{
@@ -129,9 +155,14 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
mHasNoInputSource(true),
mFastTransferExtensionAvailable(false),
mCaptureTexture(0),
mDecodedTexture(0),
mFBOTexture(0),
mDecodeFrameBuf(0),
mFullscreenVAO(0),
mGlobalParamsUBO(0),
mDecodeProgram(0),
mDecodeVertexShader(0),
mDecodeFragmentShader(0),
mProgram(0),
mVertexShader(0),
mFragmentShader(0),
@@ -195,6 +226,8 @@ OpenGLComposite::~OpenGLComposite()
glDeleteVertexArrays(1, &mFullscreenVAO);
if (mGlobalParamsUBO != 0)
glDeleteBuffers(1, &mGlobalParamsUBO);
if (mDecodeFrameBuf != 0)
glDeleteFramebuffers(1, &mDecodeFrameBuf);
if (mIdFrameBuf != 0)
glDeleteFramebuffers(1, &mIdFrameBuf);
if (mIdColorBuf != 0)
@@ -203,12 +236,15 @@ OpenGLComposite::~OpenGLComposite()
glDeleteRenderbuffers(1, &mIdDepthBuf);
if (mCaptureTexture != 0)
glDeleteTextures(1, &mCaptureTexture);
if (mDecodedTexture != 0)
glDeleteTextures(1, &mDecodedTexture);
if (mFBOTexture != 0)
glDeleteTextures(1, &mFBOTexture);
if (mUnpinnedTextureBuffer != 0)
glDeleteBuffers(1, &mUnpinnedTextureBuffer);
destroyShaderProgram();
destroyDecodeShaderProgram();
if (mControlServer)
mControlServer->Stop();
@@ -421,18 +457,52 @@ error:
void OpenGLComposite::paintGL()
{
if (!TryEnterCriticalSection(&pMutex))
{
ValidateRect(hGLWnd, NULL);
return;
}
// The DeckLink API provides IDeckLinkGLScreenPreviewHelper as a convenient way to view the playout video frames
// in a window. However, it performs a copy from host memory to the GPU which is wasteful in this case since
// we already have the rendered frame to be played out sitting in the GPU in the mIdFrameBuf frame buffer.
// Simply copy the off-screen frame buffer to on-screen frame buffer, scaling to the viewing window size.
// Copy the off-screen frame buffer to the on-screen frame buffer while preserving the
// incoming video aspect ratio. Any extra window area is cleared to black.
int destWidth = mViewWidth;
int destHeight = mViewHeight;
int destX = 0;
int destY = 0;
if (mFrameWidth > 0 && mFrameHeight > 0 && mViewWidth > 0 && mViewHeight > 0)
{
const double frameAspect = static_cast<double>(mFrameWidth) / static_cast<double>(mFrameHeight);
const double viewAspect = static_cast<double>(mViewWidth) / static_cast<double>(mViewHeight);
if (viewAspect > frameAspect)
{
destHeight = mViewHeight;
destWidth = static_cast<int>(destHeight * frameAspect + 0.5);
destX = (mViewWidth - destWidth) / 2;
}
else
{
destWidth = mViewWidth;
destHeight = static_cast<int>(destWidth / frameAspect + 0.5);
destY = (mViewHeight - destHeight) / 2;
}
}
glBindFramebuffer(GL_READ_FRAMEBUFFER, mIdFrameBuf);
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
glViewport(0, 0, mViewWidth, mViewHeight);
glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, 0, 0, mViewWidth, mViewHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glBlitFramebuffer(0, 0, mFrameWidth, mFrameHeight, destX, destY, destX + destWidth, destY + destHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
SwapBuffers(hGLDC);
ValidateRect(hGLWnd, NULL);
LeaveCriticalSection(&pMutex);
}
void OpenGLComposite::resizeGL(WORD width, WORD height)
@@ -470,6 +540,7 @@ bool OpenGLComposite::InitOpenGLState()
callbacks.updateParameter = [this](const std::string& shaderId, const std::string& parameterId, const std::string& valueJson, std::string& error) {
return UpdateParameterJson(shaderId, parameterId, valueJson, error);
};
callbacks.resetParameters = [this](const std::string& shaderId, std::string& error) { return ResetShaderParameters(shaderId, error); };
callbacks.setBypass = [this](bool bypassEnabled, std::string& error) { return SetBypassEnabled(bypassEnabled, error); };
callbacks.setMixAmount = [this](double mixAmount, std::string& error) { return SetMixAmount(mixAmount, error); };
callbacks.reloadShader = [this](std::string& error) {
@@ -490,6 +561,12 @@ bool OpenGLComposite::InitOpenGLState()
// Prepare the runtime shader program generated from the active shader package.
char compilerErrorMessage[1024];
if (! compileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage))
{
MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK);
return false;
}
if (! compileFragmentShader(sizeof(compilerErrorMessage), compilerErrorMessage))
{
MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK);
@@ -520,15 +597,34 @@ bool OpenGLComposite::InitOpenGLState()
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth/2, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glGenTextures(1, &mDecodedTexture);
glBindTexture(GL_TEXTURE_2D, mDecodedTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mFrameWidth, mFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
// Create Frame Buffer Object (FBO) to perform off-screen rendering of scene.
// This allows the render to be done on a framebuffer with width and height exactly matching the video format.
glGenFramebuffers(1, &mDecodeFrameBuf);
glGenFramebuffers(1, &mIdFrameBuf);
glGenRenderbuffers(1, &mIdColorBuf);
glGenRenderbuffers(1, &mIdDepthBuf);
glGenVertexArrays(1, &mFullscreenVAO);
glGenBuffers(1, &mGlobalParamsUBO);
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mDecodedTexture, 0);
GLenum glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize decode framebuffer."), _T("OpenGL initialization error."), MB_OK);
return false;
}
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
// Texture for FBO
@@ -549,7 +645,7 @@ bool OpenGLComposite::InitOpenGLState()
// Attach the texture which stores the playback image
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mFBOTexture, 0);
GLenum glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (glStatus != GL_FRAMEBUFFER_COMPLETE)
{
MessageBox(NULL, _T("Cannot initialize framebuffer."), _T("OpenGL initialization error."), MB_OK);
@@ -645,8 +741,18 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
wglMakeCurrent( hGLDC, hGLRC );
// Draw the effect output to the off-screen framebuffer.
const auto renderStartTime = std::chrono::steady_clock::now();
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
renderEffect();
const auto renderEndTime = std::chrono::steady_clock::now();
if (mRuntimeHost)
{
const double frameBudgetMilliseconds = mFrameTimescale != 0
? (static_cast<double>(mFrameDuration) * 1000.0) / static_cast<double>(mFrameTimescale)
: 0.0;
const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderEndTime - renderStartTime).count();
mRuntimeHost->SetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds);
}
if (mRuntimeHost)
mRuntimeHost->AdvanceFrame();
@@ -789,6 +895,58 @@ bool OpenGLComposite::ReloadShader()
return success;
}
bool OpenGLComposite::compileDecodeShader(int errorMessageSize, char* errorMessage)
{
GLsizei errorBufferSize = 0;
GLint compileResult = GL_FALSE;
GLint linkResult = GL_FALSE;
const char* vertexSource = kVertexShaderSource;
const char* fragmentSource = kDecodeFragmentShaderSource;
GLuint newVertexShader = glCreateShader(GL_VERTEX_SHADER);
glShaderSource(newVertexShader, 1, (const GLchar**)&vertexSource, NULL);
glCompileShader(newVertexShader);
glGetShaderiv(newVertexShader, GL_COMPILE_STATUS, &compileResult);
if (compileResult == GL_FALSE)
{
glGetShaderInfoLog(newVertexShader, errorMessageSize, &errorBufferSize, errorMessage);
glDeleteShader(newVertexShader);
return false;
}
GLuint newFragmentShader = glCreateShader(GL_FRAGMENT_SHADER);
glShaderSource(newFragmentShader, 1, (const GLchar**)&fragmentSource, NULL);
glCompileShader(newFragmentShader);
glGetShaderiv(newFragmentShader, GL_COMPILE_STATUS, &compileResult);
if (compileResult == GL_FALSE)
{
glGetShaderInfoLog(newFragmentShader, errorMessageSize, &errorBufferSize, errorMessage);
glDeleteShader(newVertexShader);
glDeleteShader(newFragmentShader);
return false;
}
GLuint newProgram = glCreateProgram();
glAttachShader(newProgram, newVertexShader);
glAttachShader(newProgram, newFragmentShader);
glLinkProgram(newProgram);
glGetProgramiv(newProgram, GL_LINK_STATUS, &linkResult);
if (linkResult == GL_FALSE)
{
glGetProgramInfoLog(newProgram, errorMessageSize, &errorBufferSize, errorMessage);
glDeleteProgram(newProgram);
glDeleteShader(newVertexShader);
glDeleteShader(newFragmentShader);
return false;
}
destroyDecodeShaderProgram();
mDecodeProgram = newProgram;
mDecodeVertexShader = newVertexShader;
mDecodeFragmentShader = newFragmentShader;
return true;
}
void OpenGLComposite::destroyShaderProgram()
{
if (mProgram != 0)
@@ -810,13 +968,31 @@ void OpenGLComposite::destroyShaderProgram()
}
}
void OpenGLComposite::destroyDecodeShaderProgram()
{
if (mDecodeProgram != 0)
{
glDeleteProgram(mDecodeProgram);
mDecodeProgram = 0;
}
if (mDecodeFragmentShader != 0)
{
glDeleteShader(mDecodeFragmentShader);
mDecodeFragmentShader = 0;
}
if (mDecodeVertexShader != 0)
{
glDeleteShader(mDecodeVertexShader);
mDecodeVertexShader = 0;
}
}
void OpenGLComposite::renderEffect()
{
PollRuntimeChanges();
glViewport(0, 0, mFrameWidth, mFrameHeight);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
if (mHasNoInputSource)
return;
@@ -828,8 +1004,13 @@ void OpenGLComposite::renderEffect()
glDisable(GL_BLEND);
glDisable(GL_DEPTH_TEST);
glActiveTexture(GL_TEXTURE0 + kVideoTextureUnit);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
renderDecodePass();
glBindFramebuffer(GL_FRAMEBUFFER, mIdFrameBuf);
glViewport(0, 0, mFrameWidth, mFrameHeight);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit);
glBindTexture(GL_TEXTURE_2D, mDecodedTexture);
glBindVertexArray(mFullscreenVAO);
glUseProgram(mProgram);
@@ -850,6 +1031,31 @@ void OpenGLComposite::renderEffect()
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
}
void OpenGLComposite::renderDecodePass()
{
glBindFramebuffer(GL_FRAMEBUFFER, mDecodeFrameBuf);
glViewport(0, 0, mFrameWidth, mFrameHeight);
glClear(GL_COLOR_BUFFER_BIT);
glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit);
glBindTexture(GL_TEXTURE_2D, mCaptureTexture);
glBindVertexArray(mFullscreenVAO);
glUseProgram(mDecodeProgram);
const GLint packedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uPackedVideoResolution");
const GLint decodedResolutionLocation = glGetUniformLocation(mDecodeProgram, "uDecodedVideoResolution");
if (packedResolutionLocation >= 0)
glUniform2f(packedResolutionLocation, static_cast<float>(mFrameWidth / 2), static_cast<float>(mFrameHeight));
if (decodedResolutionLocation >= 0)
glUniform2f(decodedResolutionLocation, static_cast<float>(mFrameWidth), static_cast<float>(mFrameHeight));
glDrawArrays(GL_TRIANGLES, 0, 3);
glUseProgram(0);
glBindVertexArray(0);
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE0);
}
// Compile a fullscreen shader pass from the runtime Slang source into a core-profile
// GLSL program. The renderer owns the fullscreen pass and parameter UBO layout.
bool OpenGLComposite::compileFragmentShader(int errorMessageSize, char* errorMessage)
@@ -1068,6 +1274,15 @@ bool OpenGLComposite::UpdateParameterJson(const std::string& shaderId, const std
return true;
}
bool OpenGLComposite::ResetShaderParameters(const std::string& shaderId, std::string& error)
{
if (!mRuntimeHost->ResetParameters(shaderId, error))
return false;
broadcastRuntimeState();
return true;
}
bool OpenGLComposite::SetBypassEnabled(bool bypassEnabled, std::string& error)
{
if (!mRuntimeHost->SetBypass(bypassEnabled, error))