Files
video-shader-toys/apps/LoopThroughWithOpenGLCompositing/gl/OpenGLComposite.cpp
Aiden 5fd24b3f06
Some checks failed
CI / Native Windows Build And Tests (push) Has been cancelled
CI / React UI Build (push) Has been cancelled
CI / Windows Release Package (push) Has been cancelled
Hide renderer internals
2026-05-06 10:48:50 +10:00

565 lines
19 KiB
C++

/* -LICENSE-START-
** Copyright (c) 2012 Blackmagic Design
**
** Permission is hereby granted, free of charge, to any person or organization
** obtaining a copy of the software and accompanying documentation (the
** "Software") to use, reproduce, display, distribute, sub-license, execute,
** and transmit the Software, and to prepare derivative works of the Software,
** and to permit third-parties to whom the Software is furnished to do so, in
** accordance with:
**
** (1) if the Software is obtained from Blackmagic Design, the End User License
** Agreement for the Software Development Kit ("EULA") available at
** https://www.blackmagicdesign.com/EULA/DeckLinkSDK; or
**
** (2) if the Software is obtained from any third party, such licensing terms
** as notified by that third party,
**
** and all subject to the following:
**
** (3) the copyright notices in the Software and this entire statement,
** including the above license grant, this restriction and the following
** disclaimer, must be included in all copies of the Software, in whole or in
** part, and all derivative works of the Software, unless such copies or
** derivative works are solely in the form of machine-executable object code
** generated by a source language processor.
**
** (4) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
** DEALINGS IN THE SOFTWARE.
**
** A copy of the Software is available free of charge at
** https://www.blackmagicdesign.com/desktopvideo_sdk under the EULA.
**
** -LICENSE-END-
*/
#include "ControlServer.h"
#include "DeckLinkDisplayMode.h"
#include "DeckLinkFrameTransfer.h"
#include "DeckLinkSession.h"
#include "OpenGLComposite.h"
#include "GLExtensions.h"
#include "GlRenderConstants.h"
#include "OpenGLRenderPass.h"
#include "OpenGLShaderPrograms.h"
#include "OscServer.h"
#include "RuntimeControlBridge.h"
#include <memory>
#include <string>
#include <vector>
OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC),
mDeckLink(std::make_unique<DeckLinkSession>()),
mRenderer(std::make_unique<OpenGLRenderer>())
{
InitializeCriticalSection(&pMutex);
mRuntimeHost = std::make_unique<RuntimeHost>();
mRenderPass = std::make_unique<OpenGLRenderPass>(*mRenderer);
mShaderPrograms = std::make_unique<OpenGLShaderPrograms>(*mRenderer, *mRuntimeHost);
mControlServer = std::make_unique<ControlServer>();
mOscServer = std::make_unique<OscServer>();
}
OpenGLComposite::~OpenGLComposite()
{
mDeckLink->ReleaseResources();
mRenderer->DestroyResources();
if (mOscServer)
mOscServer->Stop();
if (mControlServer)
mControlServer->Stop();
DeleteCriticalSection(&pMutex);
}
bool OpenGLComposite::InitDeckLink()
{
BMDDisplayMode inputDisplayMode = bmdModeHD1080p5994;
BMDDisplayMode outputDisplayMode = bmdModeHD1080p5994;
std::string inputDisplayModeName = "1080p59.94";
std::string outputDisplayModeName = "1080p59.94";
std::string initFailureReason;
if (mRuntimeHost && mRuntimeHost->GetRepoRoot().empty())
{
std::string runtimeError;
if (!mRuntimeHost->Initialize(runtimeError))
{
MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK);
return false;
}
}
if (mRuntimeHost)
{
if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetInputVideoFormat(), mRuntimeHost->GetInputFrameRate(), inputDisplayMode, inputDisplayModeName))
{
const std::string error = "Unsupported DeckLink inputVideoFormat/inputFrameRate in config/runtime-host.json: " +
mRuntimeHost->GetInputVideoFormat() + " / " + mRuntimeHost->GetInputFrameRate();
MessageBoxA(NULL, error.c_str(), "DeckLink input mode configuration error", MB_OK);
return false;
}
if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetOutputVideoFormat(), mRuntimeHost->GetOutputFrameRate(), outputDisplayMode, outputDisplayModeName))
{
const std::string error = "Unsupported DeckLink outputVideoFormat/outputFrameRate in config/runtime-host.json: " +
mRuntimeHost->GetOutputVideoFormat() + " / " + mRuntimeHost->GetOutputFrameRate();
MessageBoxA(NULL, error.c_str(), "DeckLink output mode configuration error", MB_OK);
return false;
}
}
if (!mDeckLink->DiscoverDevicesAndModes(inputDisplayMode, outputDisplayMode, inputDisplayModeName, outputDisplayModeName, initFailureReason))
{
const char* title = initFailureReason == "Please install the Blackmagic DeckLink drivers to use the features of this application."
? "This application requires the DeckLink drivers installed."
: "DeckLink initialization failed";
MessageBoxA(NULL, initFailureReason.c_str(), title, MB_OK | MB_ICONERROR);
return false;
}
if (! CheckOpenGLExtensions())
{
initFailureReason = "OpenGL extension checks failed.";
goto error;
}
if (mDeckLink->inputFrameWidth != mDeckLink->outputFrameWidth || mDeckLink->inputFrameHeight != mDeckLink->outputFrameHeight)
{
mRenderer->SetFastTransferAvailable(false);
OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n");
}
if (! InitOpenGLState())
{
initFailureReason = "OpenGL state initialization failed.";
goto error;
}
PublishDeckLinkOutputStatus(mDeckLink->outputModelName.empty()
? "DeckLink output device selected."
: ("Selected output device: " + mDeckLink->outputModelName));
// Resize window to match output video frame, but scale large formats down by half for viewing.
if (mDeckLink->outputFrameWidth < 1920)
resizeWindow(mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight);
else
resizeWindow(mDeckLink->outputFrameWidth / 2, mDeckLink->outputFrameHeight / 2);
if (mRenderer->FastTransferAvailable())
{
// Initialize fast video frame transfers
if (! VideoFrameTransfer::initialize(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mRenderer->CaptureTexture(), mRenderer->OutputTexture()))
{
MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK);
goto error;
}
}
if (!mDeckLink->ConfigureInput(this, hGLDC, hGLRC, inputDisplayMode, initFailureReason))
{
goto error;
}
if (!mDeckLink->input && mRuntimeHost)
{
mRuntimeHost->SetSignalStatus(false, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->inputDisplayModeName);
}
if (!mDeckLink->ConfigureOutput(this, hGLDC, hGLRC, outputDisplayMode, mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled(), initFailureReason))
{
goto error;
}
PublishDeckLinkOutputStatus(mDeckLink->statusMessage);
return true;
error:
if (!initFailureReason.empty())
MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink initialization failed", MB_OK | MB_ICONERROR);
mDeckLink->ReleaseResources();
return false;
}
void OpenGLComposite::paintGL()
{
if (!TryEnterCriticalSection(&pMutex))
{
ValidateRect(hGLWnd, NULL);
return;
}
mRenderer->PresentToWindow(hGLDC, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight);
ValidateRect(hGLWnd, NULL);
LeaveCriticalSection(&pMutex);
}
void OpenGLComposite::resizeGL(WORD width, WORD height)
{
// We don't set the project or model matrices here since the window data is copied directly from
// an off-screen FBO in paintGL(). Just save the width and height for use in paintGL().
mRenderer->ResizeView(width, height);
}
void OpenGLComposite::resizeWindow(int width, int height)
{
RECT r;
if (GetWindowRect(hGLWnd, &r))
{
SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0);
}
}
void OpenGLComposite::PublishDeckLinkOutputStatus(const std::string& statusMessage)
{
if (!mRuntimeHost)
return;
if (!statusMessage.empty())
mDeckLink->statusMessage = statusMessage;
mRuntimeHost->SetDeckLinkOutputStatus(
mDeckLink->outputModelName,
mDeckLink->supportsInternalKeying,
mDeckLink->supportsExternalKeying,
mDeckLink->keyerInterfaceAvailable,
mRuntimeHost->ExternalKeyingEnabled(),
mDeckLink->externalKeyingActive,
mDeckLink->statusMessage);
}
bool OpenGLComposite::InitOpenGLState()
{
if (! ResolveGLExtensions())
return false;
std::string runtimeError;
if (mRuntimeHost->GetRepoRoot().empty() && !mRuntimeHost->Initialize(runtimeError))
{
MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK);
return false;
}
if (!StartRuntimeControlServices(*this, *mRuntimeHost, *mControlServer, *mOscServer, runtimeError))
{
MessageBoxA(NULL, runtimeError.c_str(), "Runtime control services failed to start", MB_OK);
return false;
}
// Prepare the runtime shader program generated from the active shader package.
char compilerErrorMessage[1024];
if (!mShaderPrograms->CompileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage))
{
MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK);
return false;
}
if (!mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage))
{
MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK);
return false;
}
mShaderPrograms->ResetTemporalHistoryState();
std::string rendererError;
if (!mRenderer->InitializeResources(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, rendererError))
{
MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK);
return false;
}
broadcastRuntimeState();
return true;
}
//
// Update the captured video frame texture
//
void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource)
{
mDeckLink->hasNoInputSource = hasNoInputSource;
if (mRuntimeHost)
mRuntimeHost->SetSignalStatus(!hasNoInputSource, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->inputDisplayModeName);
if (mDeckLink->hasNoInputSource)
return; // don't transfer texture when there's no input
long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight();
IDeckLinkVideoBuffer* inputFrameBuffer = NULL;
void* videoPixels;
if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK)
return;
if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK)
{
inputFrameBuffer->Release();
return;
}
inputFrameBuffer->GetBytes(&videoPixels);
EnterCriticalSection(&pMutex);
wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread
if (mRenderer->FastTransferAvailable())
{
CComQIPtr<PinnedMemoryAllocator, &IID_PinnedMemoryAllocator> allocator(inputFrameBuffer);
if (!allocator || !allocator->transferFrame(videoPixels, mRenderer->CaptureTexture()))
OutputDebugStringA("Capture: transferFrame() failed\n");
allocator->waitForTransferComplete(videoPixels);
}
else
{
// Use a straightforward texture buffer
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer->UnpinnedTextureBuffer());
glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW);
glBindTexture(GL_TEXTURE_2D, mRenderer->CaptureTexture());
// NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mDeckLink->inputFrameWidth / 2, mDeckLink->inputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
glBindTexture(GL_TEXTURE_2D, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
}
wglMakeCurrent( NULL, NULL );
LeaveCriticalSection(&pMutex);
inputFrameBuffer->EndAccess(bmdBufferAccessRead);
inputFrameBuffer->Release();
}
// Render the live video texture through the runtime shader into the off-screen framebuffer.
// Read the result back from the frame buffer and schedule it for playout.
void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
{
EnterCriticalSection(&pMutex);
// Get the first frame from the queue
IDeckLinkMutableVideoFrame* outputVideoFrame = mDeckLink->outputVideoFrameQueue.front();
mDeckLink->outputVideoFrameQueue.push_back(outputVideoFrame);
mDeckLink->outputVideoFrameQueue.pop_front();
// make GL context current in this thread
wglMakeCurrent( hGLDC, hGLRC );
// Draw the effect output to the off-screen framebuffer.
const auto renderStartTime = std::chrono::steady_clock::now();
if (mRenderer->FastTransferAvailable())
VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->CompositeFramebuffer());
renderEffect();
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->CompositeFramebuffer());
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->OutputFramebuffer());
glBlitFramebuffer(0, 0, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, 0, 0, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR);
glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->OutputFramebuffer());
glFlush();
if (mRenderer->FastTransferAvailable())
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU);
const auto renderEndTime = std::chrono::steady_clock::now();
if (mRuntimeHost)
{
const double frameBudgetMilliseconds = mDeckLink->frameTimescale != 0
? (static_cast<double>(mDeckLink->frameDuration) * 1000.0) / static_cast<double>(mDeckLink->frameTimescale)
: 0.0;
const double renderMilliseconds = std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(renderEndTime - renderStartTime).count();
mRuntimeHost->SetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds);
}
if (mRuntimeHost)
mRuntimeHost->AdvanceFrame();
IDeckLinkVideoBuffer* outputVideoFrameBuffer;
if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK)
{
LeaveCriticalSection(&pMutex);
return;
}
if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK)
{
outputVideoFrameBuffer->Release();
LeaveCriticalSection(&pMutex);
return;
}
void* pFrame;
outputVideoFrameBuffer->GetBytes(&pFrame);
if (mRenderer->FastTransferAvailable())
{
// Finished sampling the capture texture for this frame.
if (!mDeckLink->hasNoInputSource)
VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU);
if (! mDeckLink->playoutAllocator->transferFrame(pFrame, mRenderer->OutputTexture()))
OutputDebugStringA("Playback: transferFrame() failed\n");
paintGL();
// Wait for transfer to system memory to complete
mDeckLink->playoutAllocator->waitForTransferComplete(pFrame);
}
else
{
glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->OutputFramebuffer());
glReadPixels(0, 0, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame);
paintGL();
}
outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite);
outputVideoFrameBuffer->Release();
// If the last completed frame was late or dropped, bump the scheduled time further into the future
if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped)
mDeckLink->totalPlayoutFrames += 2;
// Schedule the next frame for playout
HRESULT hr = mDeckLink->output->ScheduleVideoFrame(outputVideoFrame, (mDeckLink->totalPlayoutFrames * mDeckLink->frameDuration), mDeckLink->frameDuration, mDeckLink->frameTimescale);
if (SUCCEEDED(hr))
mDeckLink->totalPlayoutFrames++;
wglMakeCurrent( NULL, NULL );
LeaveCriticalSection(&pMutex);
}
bool OpenGLComposite::Start()
{
return mDeckLink->Start(mDeckLink->outputFrameHeight);
}
bool OpenGLComposite::Stop()
{
if (mOscServer)
mOscServer->Stop();
if (mControlServer)
mControlServer->Stop();
const bool wasExternalKeyingActive = mDeckLink->externalKeyingActive;
mDeckLink->Stop();
if (wasExternalKeyingActive)
PublishDeckLinkOutputStatus("External keying has been disabled.");
return true;
}
bool OpenGLComposite::ReloadShader()
{
char compilerErrorMessage[1024];
EnterCriticalSection(&pMutex);
wglMakeCurrent(hGLDC, hGLRC);
bool success = mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage);
if (mRuntimeHost)
mRuntimeHost->ClearReloadRequest();
wglMakeCurrent(NULL, NULL);
LeaveCriticalSection(&pMutex);
if (!success)
{
if (mRuntimeHost)
mRuntimeHost->SetCompileStatus(false, compilerErrorMessage);
MessageBoxA(NULL, compilerErrorMessage, "Slang shader reload failed", MB_OK);
}
else
{
if (mRuntimeHost)
mRuntimeHost->SetCompileStatus(true, "Shader compiled successfully.");
broadcastRuntimeState();
}
return success;
}
void OpenGLComposite::renderEffect()
{
PollRuntimeChanges();
const bool hasInputSource = !mDeckLink->hasNoInputSource;
const std::vector<RuntimeRenderState> layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight) : std::vector<RuntimeRenderState>();
const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0;
mRenderPass->Render(
hasInputSource,
layerStates,
mDeckLink->inputFrameWidth,
mDeckLink->inputFrameHeight,
historyCap,
[this](const RuntimeRenderState& state, LayerProgram::TextBinding& textBinding, std::string& error) {
return mShaderPrograms->UpdateTextBindingTexture(state, textBinding, error);
},
[this](const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) {
return mShaderPrograms->UpdateGlobalParamsBuffer(state, availableSourceHistoryLength, availableTemporalHistoryLength);
});
}
bool OpenGLComposite::PollRuntimeChanges()
{
if (!mRuntimeHost)
return true;
bool registryChanged = false;
bool reloadRequested = false;
std::string runtimeError;
if (!mRuntimeHost->PollFileChanges(registryChanged, reloadRequested, runtimeError))
{
mRuntimeHost->SetCompileStatus(false, runtimeError);
broadcastRuntimeState();
return false;
}
if (registryChanged)
broadcastRuntimeState();
if (!reloadRequested)
return true;
char compilerErrorMessage[1024] = {};
if (!mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage))
{
mRuntimeHost->SetCompileStatus(false, compilerErrorMessage);
mRuntimeHost->ClearReloadRequest();
broadcastRuntimeState();
return false;
}
mShaderPrograms->ResetTemporalHistoryState();
broadcastRuntimeState();
return true;
}
void OpenGLComposite::broadcastRuntimeState()
{
if (mControlServer)
mControlServer->BroadcastState();
}
void OpenGLComposite::resetTemporalHistoryState()
{
mShaderPrograms->ResetTemporalHistoryState();
}
bool OpenGLComposite::CheckOpenGLExtensions()
{
mRenderer->SetFastTransferAvailable(VideoFrameTransfer::checkFastMemoryTransferAvailable());
if (!mRenderer->FastTransferAvailable())
OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n");
return true;
}
////////////////////////////////////////////