/* -LICENSE-START- ** Copyright (c) 2012 Blackmagic Design ** ** Permission is hereby granted, free of charge, to any person or organization ** obtaining a copy of the software and accompanying documentation (the ** "Software") to use, reproduce, display, distribute, sub-license, execute, ** and transmit the Software, and to prepare derivative works of the Software, ** and to permit third-parties to whom the Software is furnished to do so, in ** accordance with: ** ** (1) if the Software is obtained from Blackmagic Design, the End User License ** Agreement for the Software Development Kit ("EULA") available at ** https://www.blackmagicdesign.com/EULA/DeckLinkSDK; or ** ** (2) if the Software is obtained from any third party, such licensing terms ** as notified by that third party, ** ** and all subject to the following: ** ** (3) the copyright notices in the Software and this entire statement, ** including the above license grant, this restriction and the following ** disclaimer, must be included in all copies of the Software, in whole or in ** part, and all derivative works of the Software, unless such copies or ** derivative works are solely in the form of machine-executable object code ** generated by a source language processor. ** ** (4) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT ** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE ** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ** DEALINGS IN THE SOFTWARE. ** ** A copy of the Software is available free of charge at ** https://www.blackmagicdesign.com/desktopvideo_sdk under the EULA. ** ** -LICENSE-END- */ #include "ControlServer.h" #include "DeckLinkDisplayMode.h" #include "DeckLinkFrameTransfer.h" #include "OpenGLComposite.h" #include "GLExtensions.h" #include "GlRenderConstants.h" #include "GlScopedObjects.h" #include "GlShaderSources.h" #include "OscServer.h" #include "RuntimeControlBridge.h" #include "Std140Buffer.h" #include "TextRasterizer.h" #include "TextureAssetLoader.h" #include #include #include #include #include #include #include #include namespace { void CopyErrorMessage(const std::string& message, int errorMessageSize, char* errorMessage) { if (!errorMessage || errorMessageSize <= 0) return; strncpy_s(errorMessage, errorMessageSize, message.c_str(), _TRUNCATE); } std::string TextValueForBinding(const RuntimeRenderState& state, const std::string& parameterId) { auto valueIt = state.parameterValues.find(parameterId); return valueIt == state.parameterValues.end() ? std::string() : valueIt->second.textValue; } const ShaderFontAsset* FindFontAssetForParameter(const RuntimeRenderState& state, const ShaderParameterDefinition& definition) { if (!definition.fontId.empty()) { for (const ShaderFontAsset& fontAsset : state.fontAssets) { if (fontAsset.id == definition.fontId) return &fontAsset; } } return state.fontAssets.empty() ? nullptr : &state.fontAssets.front(); } GLint FindSamplerUniformLocation(GLuint program, const std::string& samplerName) { GLint location = glGetUniformLocation(program, samplerName.c_str()); if (location >= 0) return location; return glGetUniformLocation(program, (samplerName + "_0").c_str()); } } OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), mCaptureDelegate(NULL), mPlayoutDelegate(NULL), mDLInput(NULL), mDLOutput(NULL), mDLKeyer(NULL), mPlayoutAllocator(NULL), mInputFrameWidth(0), mInputFrameHeight(0), mOutputFrameWidth(0), mOutputFrameHeight(0), mInputDisplayModeName("1080p59.94"), mOutputDisplayModeName("1080p59.94"), mHasNoInputSource(true), mDeckLinkSupportsInternalKeying(false), mDeckLinkSupportsExternalKeying(false), mDeckLinkKeyerInterfaceAvailable(false), mDeckLinkExternalKeyingActive(false), mRenderer(std::make_unique()) { InitializeCriticalSection(&pMutex); mRuntimeHost = std::make_unique(); mControlServer = std::make_unique(); mOscServer = std::make_unique(); } OpenGLComposite::~OpenGLComposite() { // Cleanup for Capture if (mDLInput != NULL) { mDLInput->SetCallback(NULL); mDLInput->Release(); mDLInput = NULL; } if (mCaptureDelegate != NULL) { mCaptureDelegate->Release(); mCaptureDelegate = NULL; } // Cleanup for Playout while (!mDLOutputVideoFrameQueue.empty()) { IDeckLinkMutableVideoFrame* frameToRelease = mDLOutputVideoFrameQueue.front(); if (frameToRelease != NULL) { frameToRelease->Release(); frameToRelease = NULL; } mDLOutputVideoFrameQueue.pop_front(); } if (mDLOutput != NULL) { if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDLKeyer->Release(); mDLKeyer = NULL; } mDLOutput->SetScheduledFrameCompletionCallback(NULL); mDLOutput->Release(); mDLOutput = NULL; } if (mPlayoutDelegate != NULL) { mPlayoutDelegate->Release(); mPlayoutDelegate = NULL; } if (mPlayoutAllocator != NULL) { mPlayoutAllocator->Release(); mPlayoutAllocator = NULL; } mRenderer->DestroyResources(); if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); DeleteCriticalSection(&pMutex); } bool OpenGLComposite::InitDeckLink() { bool bSuccess = false; IDeckLinkIterator* pDLIterator = NULL; IDeckLink* pDL = NULL; IDeckLinkProfileAttributes* deckLinkAttributes = NULL; IDeckLinkDisplayModeIterator* pDLInputDisplayModeIterator = NULL; IDeckLinkDisplayModeIterator* pDLOutputDisplayModeIterator = NULL; IDeckLinkDisplayMode* pDLInputDisplayMode = NULL; IDeckLinkDisplayMode* pDLOutputDisplayMode = NULL; BMDDisplayMode inputDisplayMode = bmdModeHD1080p5994; BMDDisplayMode outputDisplayMode = bmdModeHD1080p5994; std::string inputDisplayModeName = "1080p59.94"; std::string outputDisplayModeName = "1080p59.94"; std::string initFailureReason; int outputFrameRowBytes; HRESULT result; if (mRuntimeHost && mRuntimeHost->GetRepoRoot().empty()) { std::string runtimeError; if (!mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } } if (mRuntimeHost) { if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetInputVideoFormat(), mRuntimeHost->GetInputFrameRate(), inputDisplayMode, inputDisplayModeName)) { const std::string error = "Unsupported DeckLink inputVideoFormat/inputFrameRate in config/runtime-host.json: " + mRuntimeHost->GetInputVideoFormat() + " / " + mRuntimeHost->GetInputFrameRate(); MessageBoxA(NULL, error.c_str(), "DeckLink input mode configuration error", MB_OK); return false; } if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetOutputVideoFormat(), mRuntimeHost->GetOutputFrameRate(), outputDisplayMode, outputDisplayModeName)) { const std::string error = "Unsupported DeckLink outputVideoFormat/outputFrameRate in config/runtime-host.json: " + mRuntimeHost->GetOutputVideoFormat() + " / " + mRuntimeHost->GetOutputFrameRate(); MessageBoxA(NULL, error.c_str(), "DeckLink output mode configuration error", MB_OK); return false; } } mInputDisplayModeName = inputDisplayModeName; mOutputDisplayModeName = outputDisplayModeName; result = CoCreateInstance(CLSID_CDeckLinkIterator, NULL, CLSCTX_ALL, IID_IDeckLinkIterator, (void**)&pDLIterator); if (FAILED(result)) { MessageBox(NULL, _T("Please install the Blackmagic DeckLink drivers to use the features of this application."), _T("This application requires the DeckLink drivers installed."), MB_OK); return false; } while (pDLIterator->Next(&pDL) == S_OK) { int64_t duplexMode; bool supportsInternalKeying = false; bool supportsExternalKeying = false; std::string modelName; if (result = pDL->QueryInterface(IID_IDeckLinkProfileAttributes, (void**)&deckLinkAttributes) != S_OK) { printf("Could not obtain the IDeckLinkProfileAttributes interface - result %08x\n", result); pDL->Release(); pDL = NULL; continue; } result = deckLinkAttributes->GetInt(BMDDeckLinkDuplex, &duplexMode); BOOL attributeFlag = FALSE; if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsInternalKeying, &attributeFlag) == S_OK) supportsInternalKeying = (attributeFlag != FALSE); attributeFlag = FALSE; if (deckLinkAttributes->GetFlag(BMDDeckLinkSupportsExternalKeying, &attributeFlag) == S_OK) supportsExternalKeying = (attributeFlag != FALSE); BSTR modelNameBstr = NULL; if (deckLinkAttributes->GetString(BMDDeckLinkModelName, &modelNameBstr) == S_OK && modelNameBstr != NULL) { const int requiredBytes = WideCharToMultiByte(CP_UTF8, 0, modelNameBstr, -1, NULL, 0, NULL, NULL); if (requiredBytes > 1) { std::vector utf8Name(static_cast(requiredBytes), '\0'); if (WideCharToMultiByte(CP_UTF8, 0, modelNameBstr, -1, utf8Name.data(), requiredBytes, NULL, NULL) > 0) modelName.assign(utf8Name.data()); } SysFreeString(modelNameBstr); } deckLinkAttributes->Release(); deckLinkAttributes = NULL; if (result != S_OK || duplexMode == bmdDuplexInactive) { pDL->Release(); pDL = NULL; continue; } // Preserve the original input-then-output selection for half-duplex cards. // Input is optional later, but choosing output first can pick the wrong card. bool inputUsed = false; if (!mDLInput && pDL->QueryInterface(IID_IDeckLinkInput, (void**)&mDLInput) == S_OK) inputUsed = true; if (!mDLOutput && (!inputUsed || (duplexMode == bmdDuplexFull))) { if (pDL->QueryInterface(IID_IDeckLinkOutput, (void**)&mDLOutput) != S_OK) mDLOutput = NULL; else { mDeckLinkOutputModelName = modelName; mDeckLinkSupportsInternalKeying = supportsInternalKeying; mDeckLinkSupportsExternalKeying = supportsExternalKeying; } } pDL->Release(); pDL = NULL; if (mDLOutput && mDLInput) break; } if (!mDLOutput) { MessageBox(NULL, _T("Expected an Output DeckLink device"), _T("This application requires a DeckLink output device."), MB_OK); goto error; } if (mDLInput && mDLInput->GetDisplayModeIterator(&pDLInputDisplayModeIterator) != S_OK) { MessageBox(NULL, _T("Cannot get input Display Mode Iterator."), _T("DeckLink error."), MB_OK); goto error; } if (mDLInput && !FindDeckLinkDisplayMode(pDLInputDisplayModeIterator, inputDisplayMode, &pDLInputDisplayMode)) { const std::string error = "Cannot get specified input BMDDisplayMode for configured mode: " + inputDisplayModeName; MessageBoxA(NULL, error.c_str(), "DeckLink input error.", MB_OK); goto error; } if (pDLInputDisplayModeIterator) { pDLInputDisplayModeIterator->Release(); pDLInputDisplayModeIterator = NULL; } if (mDLOutput->GetDisplayModeIterator(&pDLOutputDisplayModeIterator) != S_OK) { MessageBox(NULL, _T("Cannot get output Display Mode Iterator."), _T("DeckLink error."), MB_OK); goto error; } if (!FindDeckLinkDisplayMode(pDLOutputDisplayModeIterator, outputDisplayMode, &pDLOutputDisplayMode)) { const std::string error = "Cannot get specified output BMDDisplayMode for configured mode: " + outputDisplayModeName; MessageBoxA(NULL, error.c_str(), "DeckLink output error.", MB_OK); goto error; } pDLOutputDisplayModeIterator->Release(); pDLOutputDisplayModeIterator = NULL; mOutputFrameWidth = pDLOutputDisplayMode->GetWidth(); mOutputFrameHeight = pDLOutputDisplayMode->GetHeight(); mInputFrameWidth = pDLInputDisplayMode ? pDLInputDisplayMode->GetWidth() : mOutputFrameWidth; mInputFrameHeight = pDLInputDisplayMode ? pDLInputDisplayMode->GetHeight() : mOutputFrameHeight; if (!mDLInput) mInputDisplayModeName = "No input - black frame"; if (! CheckOpenGLExtensions()) { initFailureReason = "OpenGL extension checks failed."; goto error; } if (mInputFrameWidth != mOutputFrameWidth || mInputFrameHeight != mOutputFrameHeight) { mRenderer->mFastTransferExtensionAvailable = false; OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n"); } if (! InitOpenGLState()) { initFailureReason = "OpenGL state initialization failed."; goto error; } if (mRuntimeHost) { mDeckLinkStatusMessage = mDeckLinkOutputModelName.empty() ? "DeckLink output device selected." : ("Selected output device: " + mDeckLinkOutputModelName); mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, mDeckLinkStatusMessage); } pDLOutputDisplayMode->GetFrameRate(&mFrameDuration, &mFrameTimescale); // Resize window to match output video frame, but scale large formats down by half for viewing. if (mOutputFrameWidth < 1920) resizeWindow(mOutputFrameWidth, mOutputFrameHeight); else resizeWindow(mOutputFrameWidth / 2, mOutputFrameHeight / 2); if (mRenderer->mFastTransferExtensionAvailable) { // Initialize fast video frame transfers if (! VideoFrameTransfer::initialize(mInputFrameWidth, mInputFrameHeight, mRenderer->mCaptureTexture, mRenderer->mOutputTexture)) { MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK); goto error; } } if (mDLInput) { // Use custom allocators so we pin only once then recycle them CComPtr captureAllocator(new (std::nothrow) InputAllocatorPool(hGLDC, hGLRC)); if (mDLInput->EnableVideoInputWithAllocatorProvider(inputDisplayMode, bmdFormat8BitYUV, bmdVideoInputFlagDefault, captureAllocator) != S_OK) { OutputDebugStringA("DeckLink input could not be enabled; continuing in output-only black-frame mode.\n"); mDLInput->Release(); mDLInput = NULL; mHasNoInputSource = true; mInputDisplayModeName = "No input - black frame"; if (mRuntimeHost) mRuntimeHost->SetSignalStatus(false, mInputFrameWidth, mInputFrameHeight, mInputDisplayModeName); } } if (mDLInput) { mCaptureDelegate = new CaptureDelegate(this); if (mDLInput->SetCallback(mCaptureDelegate) != S_OK) { initFailureReason = "DeckLink input setup failed while installing the capture callback."; goto error; } } else if (mRuntimeHost) { mRuntimeHost->SetSignalStatus(false, mInputFrameWidth, mInputFrameHeight, mInputDisplayModeName); } if (mDLOutput->RowBytesForPixelFormat(bmdFormat8BitBGRA, mOutputFrameWidth, &outputFrameRowBytes) != S_OK) { initFailureReason = "DeckLink output setup failed while calculating BGRA row bytes."; goto error; } // Use a custom allocator so we pin only once then recycle them mPlayoutAllocator = new PinnedMemoryAllocator(hGLDC, hGLRC, VideoFrameTransfer::GPUtoCPU, 1, outputFrameRowBytes * mOutputFrameHeight); if (mDLOutput->EnableVideoOutput(outputDisplayMode, bmdVideoOutputFlagDefault) != S_OK) { initFailureReason = "DeckLink output setup failed while enabling video output."; goto error; } if (mDLOutput->QueryInterface(IID_IDeckLinkKeyer, (void**)&mDLKeyer) == S_OK && mDLKeyer != NULL) mDeckLinkKeyerInterfaceAvailable = true; if (mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled()) { if (!mDeckLinkSupportsExternalKeying) { mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not report external keying support."; } else if (!mDeckLinkKeyerInterfaceAvailable) { mDeckLinkStatusMessage = "External keying was requested, but the selected DeckLink output does not expose the IDeckLinkKeyer interface."; } else if (mDLKeyer->Enable(TRUE) != S_OK || mDLKeyer->SetLevel(255) != S_OK) { mDeckLinkStatusMessage = "External keying was requested, but enabling the DeckLink keyer failed."; } else { mDeckLinkExternalKeyingActive = true; mDeckLinkStatusMessage = "External keying is active on the selected DeckLink output."; } } else if (mDeckLinkSupportsExternalKeying) { mDeckLinkStatusMessage = "Selected DeckLink output supports external keying. Set enableExternalKeying to true in runtime-host.json to request it."; } if (mRuntimeHost) { mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, mDeckLinkStatusMessage); } // Create a queue of 10 IDeckLinkMutableVideoFrame objects to use for scheduling output video frames. // The ScheduledFrameCompleted() callback will immediately schedule a new frame using the next video frame from this queue. for (int i = 0; i < 10; i++) { // The frame read back from the GPU frame buffer and used for the playout video frame is in BGRA format. // The BGRA frame will be converted on playout to YCbCr either in hardware on most DeckLink cards or in software // within the DeckLink API for DeckLink devices without this hardware conversion. // If you want RGB 4:4:4 format to be played out "over the wire" in SDI, turn on the "Use 4:4:4 SDI" in the control // panel or turn on the bmdDeckLinkConfig444SDIVideoOutput flag using the IDeckLinkConfiguration interface. IDeckLinkMutableVideoFrame* outputFrame; IDeckLinkVideoBuffer* outputFrameBuffer = NULL; if (mPlayoutAllocator->AllocateVideoBuffer(&outputFrameBuffer) != S_OK) { initFailureReason = "DeckLink output setup failed while allocating an output frame buffer."; goto error; } if (mDLOutput->CreateVideoFrameWithBuffer(mOutputFrameWidth, mOutputFrameHeight, outputFrameRowBytes, bmdFormat8BitBGRA, bmdFrameFlagFlipVertical, outputFrameBuffer, &outputFrame) != S_OK) { initFailureReason = "DeckLink output setup failed while creating an output video frame."; goto error; } mDLOutputVideoFrameQueue.push_back(outputFrame); } mPlayoutDelegate = new PlayoutDelegate(this); if (mPlayoutDelegate == NULL) { initFailureReason = "DeckLink output setup failed while creating the playout callback."; goto error; } if (mDLOutput->SetScheduledFrameCompletionCallback(mPlayoutDelegate) != S_OK) { initFailureReason = "DeckLink output setup failed while installing the scheduled-frame callback."; goto error; } bSuccess = true; error: if (!bSuccess) { if (!initFailureReason.empty()) MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink initialization failed", MB_OK | MB_ICONERROR); if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDLKeyer->Release(); mDLKeyer = NULL; mDeckLinkExternalKeyingActive = false; } if (mDLInput != NULL) { mDLInput->Release(); mDLInput = NULL; } if (mDLOutput != NULL) { mDLOutput->Release(); mDLOutput = NULL; } } if (pDL != NULL) { pDL->Release(); pDL = NULL; } if (pDLInputDisplayMode != NULL) { pDLInputDisplayMode->Release(); pDLInputDisplayMode = NULL; } if (pDLOutputDisplayMode != NULL) { pDLOutputDisplayMode->Release(); pDLOutputDisplayMode = NULL; } if (pDLInputDisplayModeIterator != NULL) { pDLInputDisplayModeIterator->Release(); pDLInputDisplayModeIterator = NULL; } if (pDLOutputDisplayModeIterator != NULL) { pDLOutputDisplayModeIterator->Release(); pDLOutputDisplayModeIterator = NULL; } if (pDLIterator != NULL) { pDLIterator->Release(); pDLIterator = NULL; } return bSuccess; } void OpenGLComposite::paintGL() { if (!TryEnterCriticalSection(&pMutex)) { ValidateRect(hGLWnd, NULL); return; } mRenderer->PresentToWindow(hGLDC, mOutputFrameWidth, mOutputFrameHeight); ValidateRect(hGLWnd, NULL); LeaveCriticalSection(&pMutex); } void OpenGLComposite::resizeGL(WORD width, WORD height) { // We don't set the project or model matrices here since the window data is copied directly from // an off-screen FBO in paintGL(). Just save the width and height for use in paintGL(). mRenderer->ResizeView(width, height); } void OpenGLComposite::resizeWindow(int width, int height) { RECT r; if (GetWindowRect(hGLWnd, &r)) { SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0); } } bool OpenGLComposite::InitOpenGLState() { if (! ResolveGLExtensions()) return false; std::string runtimeError; if (mRuntimeHost->GetRepoRoot().empty() && !mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } if (!StartRuntimeControlServices(*this, *mRuntimeHost, *mControlServer, *mOscServer, runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime control services failed to start", MB_OK); return false; } // Prepare the runtime shader program generated from the active shader package. char compilerErrorMessage[1024]; if (! compileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK); return false; } if (! compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK); return false; } resetTemporalHistoryState(); std::string rendererError; if (!mRenderer->InitializeResources(mInputFrameWidth, mInputFrameHeight, mOutputFrameWidth, mOutputFrameHeight, rendererError)) { MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK); return false; } broadcastRuntimeState(); return true; } // // Update the captured video frame texture // void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource) { mHasNoInputSource = hasNoInputSource; if (mRuntimeHost) mRuntimeHost->SetSignalStatus(!hasNoInputSource, mInputFrameWidth, mInputFrameHeight, mInputDisplayModeName); if (mHasNoInputSource) return; // don't transfer texture when there's no input long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight(); IDeckLinkVideoBuffer* inputFrameBuffer = NULL; void* videoPixels; if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK) return; if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK) { inputFrameBuffer->Release(); return; } inputFrameBuffer->GetBytes(&videoPixels); EnterCriticalSection(&pMutex); wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread if (mRenderer->mFastTransferExtensionAvailable) { CComQIPtr allocator(inputFrameBuffer); if (!allocator || !allocator->transferFrame(videoPixels, mRenderer->mCaptureTexture)) OutputDebugStringA("Capture: transferFrame() failed\n"); allocator->waitForTransferComplete(videoPixels); } else { // Use a straightforward texture buffer glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer->mUnpinnedTextureBuffer); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW); glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture); // NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mInputFrameWidth / 2, mInputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); } wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); inputFrameBuffer->EndAccess(bmdBufferAccessRead); inputFrameBuffer->Release(); } // Render the live video texture through the runtime shader into the off-screen framebuffer. // Read the result back from the frame buffer and schedule it for playout. void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult) { EnterCriticalSection(&pMutex); // Get the first frame from the queue IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // make GL context current in this thread wglMakeCurrent( hGLDC, hGLRC ); // Draw the effect output to the off-screen framebuffer. const auto renderStartTime = std::chrono::steady_clock::now(); if (mRenderer->mFastTransferExtensionAvailable) VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf); renderEffect(); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mIdFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glFlush(); if (mRenderer->mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU); const auto renderEndTime = std::chrono::steady_clock::now(); if (mRuntimeHost) { const double frameBudgetMilliseconds = mFrameTimescale != 0 ? (static_cast(mFrameDuration) * 1000.0) / static_cast(mFrameTimescale) : 0.0; const double renderMilliseconds = std::chrono::duration_cast>(renderEndTime - renderStartTime).count(); mRuntimeHost->SetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds); } if (mRuntimeHost) mRuntimeHost->AdvanceFrame(); IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) { LeaveCriticalSection(&pMutex); return; } if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); LeaveCriticalSection(&pMutex); return; } void* pFrame; outputVideoFrameBuffer->GetBytes(&pFrame); if (mRenderer->mFastTransferExtensionAvailable) { // Finished with mRenderer->mCaptureTexture if (!mHasNoInputSource) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); if (! mPlayoutAllocator->transferFrame(pFrame, mRenderer->mOutputTexture)) OutputDebugStringA("Playback: transferFrame() failed\n"); paintGL(); // Wait for transfer to system memory to complete mPlayoutAllocator->waitForTransferComplete(pFrame); } else { glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glReadPixels(0, 0, mOutputFrameWidth, mOutputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); paintGL(); } outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); // If the last completed frame was late or dropped, bump the scheduled time further into the future if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped) mTotalPlayoutFrames += 2; // Schedule the next frame for playout HRESULT hr = mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale); if (SUCCEEDED(hr)) mTotalPlayoutFrames++; wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); } bool OpenGLComposite::Start() { mTotalPlayoutFrames = 0; if (!mDLOutput) { MessageBoxA(NULL, "Cannot start playout because no DeckLink output device is available.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } if (mDLOutputVideoFrameQueue.empty()) { MessageBoxA(NULL, "Cannot start playout because the output frame queue is empty.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } // Preroll frames for (unsigned i = 0; i < kPrerollFrameCount; i++) { // Take each video frame from the front of the queue and move it to the back IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front(); mDLOutputVideoFrameQueue.push_back(outputVideoFrame); mDLOutputVideoFrameQueue.pop_front(); // Start with a black frame for playout IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) { MessageBoxA(NULL, "Could not query the preroll output frame buffer.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); MessageBoxA(NULL, "Could not write to the preroll output frame buffer.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } void* pFrame; outputVideoFrameBuffer->GetBytes((void**)&pFrame); memset(pFrame, 0, outputVideoFrame->GetRowBytes() * mOutputFrameHeight); // 0 is black in BGRA format outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); if (mDLOutput->ScheduleVideoFrame(outputVideoFrame, (mTotalPlayoutFrames * mFrameDuration), mFrameDuration, mFrameTimescale) != S_OK) { MessageBoxA(NULL, "Could not schedule a preroll output frame.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } mTotalPlayoutFrames++; } if (mDLInput) { if (mDLInput->StartStreams() != S_OK) { MessageBoxA(NULL, "Could not start the DeckLink input stream.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } } if (mDLOutput->StartScheduledPlayback(0, mFrameTimescale, 1.0) != S_OK) { MessageBoxA(NULL, "Could not start DeckLink scheduled playback.", "DeckLink start failed", MB_OK | MB_ICONERROR); return false; } return true; } bool OpenGLComposite::Stop() { if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); if (mDLKeyer != NULL) { mDLKeyer->Disable(); mDeckLinkExternalKeyingActive = false; if (mRuntimeHost) { mRuntimeHost->SetDeckLinkOutputStatus( mDeckLinkOutputModelName, mDeckLinkSupportsInternalKeying, mDeckLinkSupportsExternalKeying, mDeckLinkKeyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLinkExternalKeyingActive, "External keying has been disabled."); } } if (mDLInput) { mDLInput->StopStreams(); mDLInput->DisableVideoInput(); } if (mDLOutput) { mDLOutput->StopScheduledPlayback(0, NULL, 0); mDLOutput->DisableVideoOutput(); } return true; } bool OpenGLComposite::ReloadShader() { char compilerErrorMessage[1024]; EnterCriticalSection(&pMutex); wglMakeCurrent(hGLDC, hGLRC); bool success = compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage); if (mRuntimeHost) mRuntimeHost->ClearReloadRequest(); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&pMutex); if (!success) { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); MessageBoxA(NULL, compilerErrorMessage, "Slang shader reload failed", MB_OK); } else { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(true, "Shader compiled successfully."); broadcastRuntimeState(); } return success; } bool OpenGLComposite::compileSingleLayerProgram(const RuntimeRenderState& state, LayerProgram& layerProgram, int errorMessageSize, char* errorMessage) { GLsizei errorBufferSize = 0; GLint compileResult = GL_FALSE; GLint linkResult = GL_FALSE; std::string fragmentShaderSource; std::string loadError; std::vector textureBindings; const char* vertexSource = kFullscreenTriangleVertexShaderSource; if (!mRuntimeHost->BuildLayerFragmentShaderSource(state.layerId, fragmentShaderSource, loadError)) { CopyErrorMessage(loadError, errorMessageSize, errorMessage); return false; } const char* fragmentSource = fragmentShaderSource.c_str(); ScopedGlShader newVertexShader(glCreateShader(GL_VERTEX_SHADER)); glShaderSource(newVertexShader.get(), 1, (const GLchar**)&vertexSource, NULL); glCompileShader(newVertexShader.get()); glGetShaderiv(newVertexShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newVertexShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlShader newFragmentShader(glCreateShader(GL_FRAGMENT_SHADER)); glShaderSource(newFragmentShader.get(), 1, (const GLchar**)&fragmentSource, NULL); glCompileShader(newFragmentShader.get()); glGetShaderiv(newFragmentShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newFragmentShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlProgram newProgram(glCreateProgram()); glAttachShader(newProgram.get(), newVertexShader.get()); glAttachShader(newProgram.get(), newFragmentShader.get()); glLinkProgram(newProgram.get()); glGetProgramiv(newProgram.get(), GL_LINK_STATUS, &linkResult); if (linkResult == GL_FALSE) { glGetProgramInfoLog(newProgram.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } for (const ShaderTextureAsset& textureAsset : state.textureAssets) { LayerProgram::TextureBinding textureBinding; textureBinding.samplerName = textureAsset.id; textureBinding.sourcePath = textureAsset.path; if (!loadTextureAsset(textureAsset, textureBinding.texture, loadError)) { for (LayerProgram::TextureBinding& loadedTexture : textureBindings) { if (loadedTexture.texture != 0) glDeleteTextures(1, &loadedTexture.texture); } CopyErrorMessage(loadError, errorMessageSize, errorMessage); return false; } textureBindings.push_back(textureBinding); } std::vector textBindings; for (const ShaderParameterDefinition& definition : state.parameterDefinitions) { if (definition.type != ShaderParameterType::Text) continue; LayerProgram::TextBinding textBinding; textBinding.parameterId = definition.id; textBinding.samplerName = definition.id + "Texture"; textBinding.fontId = definition.fontId; glGenTextures(1, &textBinding.texture); glBindTexture(GL_TEXTURE_2D, textBinding.texture); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); std::vector empty(static_cast(kTextTextureWidth) * kTextTextureHeight * 4, 0); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, kTextTextureWidth, kTextTextureHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, empty.data()); glBindTexture(GL_TEXTURE_2D, 0); textBindings.push_back(textBinding); } const GLuint globalParamsIndex = glGetUniformBlockIndex(newProgram.get(), "GlobalParams"); if (globalParamsIndex != GL_INVALID_INDEX) glUniformBlockBinding(newProgram.get(), globalParamsIndex, kGlobalParamsBindingPoint); const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; const GLuint shaderTextureBase = state.isTemporal ? kSourceHistoryTextureUnitBase + historyCap + historyCap : kSourceHistoryTextureUnitBase; glUseProgram(newProgram.get()); const GLint videoInputLocation = glGetUniformLocation(newProgram.get(), "gVideoInput"); if (videoInputLocation >= 0) glUniform1i(videoInputLocation, static_cast(kDecodedVideoTextureUnit)); for (unsigned index = 0; index < historyCap; ++index) { const std::string sourceSamplerName = "gSourceHistory" + std::to_string(index); const GLint sourceSamplerLocation = glGetUniformLocation(newProgram.get(), sourceSamplerName.c_str()); if (sourceSamplerLocation >= 0) glUniform1i(sourceSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + index)); const std::string temporalSamplerName = "gTemporalHistory" + std::to_string(index); const GLint temporalSamplerLocation = glGetUniformLocation(newProgram.get(), temporalSamplerName.c_str()); if (temporalSamplerLocation >= 0) glUniform1i(temporalSamplerLocation, static_cast(kSourceHistoryTextureUnitBase + historyCap + index)); } for (std::size_t index = 0; index < textureBindings.size(); ++index) { const GLint textureSamplerLocation = FindSamplerUniformLocation(newProgram.get(), textureBindings[index].samplerName); if (textureSamplerLocation >= 0) glUniform1i(textureSamplerLocation, static_cast(shaderTextureBase + static_cast(index))); } const GLuint textTextureBase = shaderTextureBase + static_cast(textureBindings.size()); for (std::size_t index = 0; index < textBindings.size(); ++index) { const GLint textSamplerLocation = FindSamplerUniformLocation(newProgram.get(), textBindings[index].samplerName); if (textSamplerLocation >= 0) glUniform1i(textSamplerLocation, static_cast(textTextureBase + static_cast(index))); } glUseProgram(0); layerProgram.layerId = state.layerId; layerProgram.shaderId = state.shaderId; layerProgram.shaderTextureBase = shaderTextureBase; layerProgram.program = newProgram.release(); layerProgram.vertexShader = newVertexShader.release(); layerProgram.fragmentShader = newFragmentShader.release(); layerProgram.textureBindings.swap(textureBindings); layerProgram.textBindings.swap(textBindings); return true; } bool OpenGLComposite::compileLayerPrograms(int errorMessageSize, char* errorMessage) { const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mInputFrameWidth, mInputFrameHeight) : std::vector(); std::string temporalError; if (!validateTemporalTextureUnitBudget(layerStates, temporalError)) { CopyErrorMessage(temporalError, errorMessageSize, errorMessage); return false; } if (!ensureTemporalHistoryResources(layerStates, temporalError)) { CopyErrorMessage(temporalError, errorMessageSize, errorMessage); return false; } std::vector newPrograms; newPrograms.reserve(layerStates.size()); for (const RuntimeRenderState& state : layerStates) { LayerProgram layerProgram; if (!compileSingleLayerProgram(state, layerProgram, errorMessageSize, errorMessage)) { for (LayerProgram& program : newPrograms) destroySingleLayerProgram(program); return false; } newPrograms.push_back(layerProgram); } destroyLayerPrograms(); mRenderer->mLayerPrograms.swap(newPrograms); if (mRuntimeHost) { mRuntimeHost->SetCompileStatus(true, "Shader layers compiled successfully."); mRuntimeHost->ClearReloadRequest(); } return true; } bool OpenGLComposite::compileDecodeShader(int errorMessageSize, char* errorMessage) { GLsizei errorBufferSize = 0; GLint compileResult = GL_FALSE; GLint linkResult = GL_FALSE; const char* vertexSource = kFullscreenTriangleVertexShaderSource; const char* fragmentSource = kDecodeFragmentShaderSource; ScopedGlShader newVertexShader(glCreateShader(GL_VERTEX_SHADER)); glShaderSource(newVertexShader.get(), 1, (const GLchar**)&vertexSource, NULL); glCompileShader(newVertexShader.get()); glGetShaderiv(newVertexShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newVertexShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlShader newFragmentShader(glCreateShader(GL_FRAGMENT_SHADER)); glShaderSource(newFragmentShader.get(), 1, (const GLchar**)&fragmentSource, NULL); glCompileShader(newFragmentShader.get()); glGetShaderiv(newFragmentShader.get(), GL_COMPILE_STATUS, &compileResult); if (compileResult == GL_FALSE) { glGetShaderInfoLog(newFragmentShader.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } ScopedGlProgram newProgram(glCreateProgram()); glAttachShader(newProgram.get(), newVertexShader.get()); glAttachShader(newProgram.get(), newFragmentShader.get()); glLinkProgram(newProgram.get()); glGetProgramiv(newProgram.get(), GL_LINK_STATUS, &linkResult); if (linkResult == GL_FALSE) { glGetProgramInfoLog(newProgram.get(), errorMessageSize, &errorBufferSize, errorMessage); return false; } destroyDecodeShaderProgram(); mRenderer->mDecodeProgram = newProgram.release(); mRenderer->mDecodeVertexShader = newVertexShader.release(); mRenderer->mDecodeFragmentShader = newFragmentShader.release(); return true; } void OpenGLComposite::destroySingleLayerProgram(LayerProgram& layerProgram) { mRenderer->DestroySingleLayerProgram(layerProgram); } void OpenGLComposite::destroyLayerPrograms() { mRenderer->DestroyLayerPrograms(); } bool OpenGLComposite::loadTextureAsset(const ShaderTextureAsset& textureAsset, GLuint& textureId, std::string& error) { return LoadTextureAsset(textureAsset, textureId, error); } bool OpenGLComposite::renderTextBindingTexture(const RuntimeRenderState& state, LayerProgram::TextBinding& textBinding, std::string& error) { const std::string text = TextValueForBinding(state, textBinding.parameterId); if (text == textBinding.renderedText && textBinding.renderedWidth == kTextTextureWidth && textBinding.renderedHeight == kTextTextureHeight) return true; auto definitionIt = std::find_if(state.parameterDefinitions.begin(), state.parameterDefinitions.end(), [&textBinding](const ShaderParameterDefinition& definition) { return definition.id == textBinding.parameterId; }); if (definitionIt == state.parameterDefinitions.end()) return true; const ShaderFontAsset* fontAsset = FindFontAssetForParameter(state, *definitionIt); std::filesystem::path fontPath; if (fontAsset) fontPath = fontAsset->path; std::vector sdf; if (!RasterizeTextSdf(text, fontPath, sdf, error)) return false; GLint previousActiveTexture = 0; GLint previousUnpackBuffer = 0; glGetIntegerv(GL_ACTIVE_TEXTURE, &previousActiveTexture); glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpackBuffer); glActiveTexture(GL_TEXTURE0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); glBindTexture(GL_TEXTURE_2D, textBinding.texture); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, kTextTextureWidth, kTextTextureHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, sdf.data()); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, static_cast(previousUnpackBuffer)); glActiveTexture(static_cast(previousActiveTexture)); textBinding.renderedText = text; textBinding.renderedWidth = kTextTextureWidth; textBinding.renderedHeight = kTextTextureHeight; return true; } void OpenGLComposite::bindLayerTextureAssets(const LayerProgram& layerProgram) { const GLuint shaderTextureBase = layerProgram.shaderTextureBase != 0 ? layerProgram.shaderTextureBase : kSourceHistoryTextureUnitBase; for (std::size_t index = 0; index < layerProgram.textureBindings.size(); ++index) { glActiveTexture(GL_TEXTURE0 + shaderTextureBase + static_cast(index)); glBindTexture(GL_TEXTURE_2D, layerProgram.textureBindings[index].texture); } const GLuint textTextureBase = shaderTextureBase + static_cast(layerProgram.textureBindings.size()); for (std::size_t index = 0; index < layerProgram.textBindings.size(); ++index) { glActiveTexture(GL_TEXTURE0 + textTextureBase + static_cast(index)); glBindTexture(GL_TEXTURE_2D, layerProgram.textBindings[index].texture); } glActiveTexture(GL_TEXTURE0); } void OpenGLComposite::destroyDecodeShaderProgram() { mRenderer->DestroyDecodeShaderProgram(); } bool OpenGLComposite::validateTemporalTextureUnitBudget(const std::vector& layerStates, std::string& error) const { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; return mRenderer->mTemporalHistory.ValidateTextureUnitBudget(layerStates, historyCap, error); } void OpenGLComposite::resetTemporalHistoryState() { mRenderer->mTemporalHistory.ResetState(); } bool OpenGLComposite::ensureTemporalHistoryResources(const std::vector& layerStates, std::string& error) { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; return mRenderer->mTemporalHistory.EnsureResources(layerStates, historyCap, mInputFrameWidth, mInputFrameHeight, error); } unsigned OpenGLComposite::sourceHistoryAvailableCount() const { return mRenderer->mTemporalHistory.SourceAvailableCount(); } unsigned OpenGLComposite::temporalHistoryAvailableCountForLayer(const std::string& layerId) const { return mRenderer->mTemporalHistory.AvailableCountForLayer(layerId); } void OpenGLComposite::bindHistorySamplers(const RuntimeRenderState& state, GLuint currentSourceTexture) { const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; mRenderer->mTemporalHistory.BindSamplers(state, currentSourceTexture, historyCap); } void OpenGLComposite::renderEffect() { PollRuntimeChanges(); const bool hasInputSource = !mHasNoInputSource; if (hasInputSource && mRenderer->mFastTransferExtensionAvailable) { // Signal that we're about to draw using mRenderer->mCaptureTexture onto mRenderer->mFBOTexture. VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::CPUtoGPU); } glDisable(GL_BLEND); glDisable(GL_DEPTH_TEST); if (hasInputSource) { renderDecodePass(); } else { glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf); glViewport(0, 0, mInputFrameWidth, mInputFrameHeight); glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClear(GL_COLOR_BUFFER_BIT); } const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mInputFrameWidth, mInputFrameHeight) : std::vector(); if (layerStates.empty() || mRenderer->mLayerPrograms.empty()) { glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mDecodeFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mIdFrameBuf); glBlitFramebuffer(0, 0, mInputFrameWidth, mInputFrameHeight, 0, 0, mInputFrameWidth, mInputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf); } else { GLuint sourceTexture = mRenderer->mDecodedTexture; GLuint sourceFrameBuffer = mRenderer->mDecodeFrameBuf; for (std::size_t index = 0; index < layerStates.size() && index < mRenderer->mLayerPrograms.size(); ++index) { const std::size_t remaining = layerStates.size() - index; const bool writeToMain = (remaining % 2) == 1; renderShaderProgram(sourceTexture, writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf, mRenderer->mLayerPrograms[index], layerStates[index]); if (layerStates[index].temporalHistorySource == TemporalHistorySource::PreLayerInput) mRenderer->mTemporalHistory.PushPreLayerFramebuffer(layerStates[index].layerId, sourceFrameBuffer, mInputFrameWidth, mInputFrameHeight); sourceTexture = writeToMain ? mRenderer->mFBOTexture : mRenderer->mLayerTempTexture; sourceFrameBuffer = writeToMain ? mRenderer->mIdFrameBuf : mRenderer->mLayerTempFrameBuf; } } mRenderer->mTemporalHistory.PushSourceFramebuffer(mRenderer->mDecodeFrameBuf, mInputFrameWidth, mInputFrameHeight); if (hasInputSource && mRenderer->mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); } void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinationFrameBuffer, LayerProgram& layerProgram, const RuntimeRenderState& state) { for (LayerProgram::TextBinding& textBinding : layerProgram.textBindings) { std::string textError; if (!renderTextBindingTexture(state, textBinding, textError)) OutputDebugStringA((textError + "\n").c_str()); } glBindFramebuffer(GL_FRAMEBUFFER, destinationFrameBuffer); glViewport(0, 0, mInputFrameWidth, mInputFrameHeight); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, sourceTexture); bindHistorySamplers(state, sourceTexture); bindLayerTextureAssets(layerProgram); glBindVertexArray(mRenderer->mFullscreenVAO); glUseProgram(layerProgram.program); updateGlobalParamsBuffer(state, sourceHistoryAvailableCount(), temporalHistoryAvailableCountForLayer(state.layerId)); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindVertexArray(0); const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; for (unsigned index = 0; index < historyCap; ++index) { glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + index); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0 + kSourceHistoryTextureUnitBase + historyCap + index); glBindTexture(GL_TEXTURE_2D, 0); } const GLuint shaderTextureBase = layerProgram.shaderTextureBase != 0 ? layerProgram.shaderTextureBase : kSourceHistoryTextureUnitBase; for (std::size_t index = 0; index < layerProgram.textureBindings.size() + layerProgram.textBindings.size(); ++index) { glActiveTexture(GL_TEXTURE0 + shaderTextureBase + static_cast(index)); glBindTexture(GL_TEXTURE_2D, 0); } glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); } void OpenGLComposite::renderDecodePass() { glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mDecodeFrameBuf); glViewport(0, 0, mInputFrameWidth, mInputFrameHeight); glClear(GL_COLOR_BUFFER_BIT); glActiveTexture(GL_TEXTURE0 + kPackedVideoTextureUnit); glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture); glBindVertexArray(mRenderer->mFullscreenVAO); glUseProgram(mRenderer->mDecodeProgram); const GLint packedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uPackedVideoResolution"); const GLint decodedResolutionLocation = glGetUniformLocation(mRenderer->mDecodeProgram, "uDecodedVideoResolution"); if (packedResolutionLocation >= 0) glUniform2f(packedResolutionLocation, static_cast(mInputFrameWidth / 2), static_cast(mInputFrameHeight)); if (decodedResolutionLocation >= 0) glUniform2f(decodedResolutionLocation, static_cast(mInputFrameWidth), static_cast(mInputFrameHeight)); glDrawArrays(GL_TRIANGLES, 0, 3); glUseProgram(0); glBindVertexArray(0); glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); } bool OpenGLComposite::PollRuntimeChanges() { if (!mRuntimeHost) return true; bool registryChanged = false; bool reloadRequested = false; std::string runtimeError; if (!mRuntimeHost->PollFileChanges(registryChanged, reloadRequested, runtimeError)) { mRuntimeHost->SetCompileStatus(false, runtimeError); broadcastRuntimeState(); return false; } if (registryChanged) broadcastRuntimeState(); if (!reloadRequested) return true; char compilerErrorMessage[1024] = {}; if (!compileLayerPrograms(sizeof(compilerErrorMessage), compilerErrorMessage)) { mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); mRuntimeHost->ClearReloadRequest(); broadcastRuntimeState(); return false; } resetTemporalHistoryState(); broadcastRuntimeState(); return true; } void OpenGLComposite::broadcastRuntimeState() { if (mControlServer) mControlServer->BroadcastState(); } bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) { std::vector buffer; buffer.reserve(512); AppendStd140Float(buffer, static_cast(state.timeSeconds)); AppendStd140Vec2(buffer, static_cast(state.inputWidth), static_cast(state.inputHeight)); AppendStd140Vec2(buffer, static_cast(state.outputWidth), static_cast(state.outputHeight)); AppendStd140Float(buffer, static_cast(state.frameCount)); AppendStd140Float(buffer, static_cast(state.mixAmount)); AppendStd140Float(buffer, static_cast(state.bypass)); const unsigned effectiveSourceHistoryLength = availableSourceHistoryLength < state.effectiveTemporalHistoryLength ? availableSourceHistoryLength : state.effectiveTemporalHistoryLength; const unsigned effectiveTemporalHistoryLength = (state.temporalHistorySource == TemporalHistorySource::PreLayerInput) ? (availableTemporalHistoryLength < state.effectiveTemporalHistoryLength ? availableTemporalHistoryLength : state.effectiveTemporalHistoryLength) : 0u; AppendStd140Int(buffer, static_cast(effectiveSourceHistoryLength)); AppendStd140Int(buffer, static_cast(effectiveTemporalHistoryLength)); for (const ShaderParameterDefinition& definition : state.parameterDefinitions) { auto valueIt = state.parameterValues.find(definition.id); const ShaderParameterValue value = valueIt != state.parameterValues.end() ? valueIt->second : ShaderParameterValue(); switch (definition.type) { case ShaderParameterType::Float: AppendStd140Float(buffer, value.numberValues.empty() ? 0.0f : static_cast(value.numberValues[0])); break; case ShaderParameterType::Vec2: AppendStd140Vec2(buffer, value.numberValues.size() > 0 ? static_cast(value.numberValues[0]) : 0.0f, value.numberValues.size() > 1 ? static_cast(value.numberValues[1]) : 0.0f); break; case ShaderParameterType::Color: AppendStd140Vec4(buffer, value.numberValues.size() > 0 ? static_cast(value.numberValues[0]) : 1.0f, value.numberValues.size() > 1 ? static_cast(value.numberValues[1]) : 1.0f, value.numberValues.size() > 2 ? static_cast(value.numberValues[2]) : 1.0f, value.numberValues.size() > 3 ? static_cast(value.numberValues[3]) : 1.0f); break; case ShaderParameterType::Boolean: AppendStd140Int(buffer, value.booleanValue ? 1 : 0); break; case ShaderParameterType::Enum: { int selectedIndex = 0; for (std::size_t optionIndex = 0; optionIndex < definition.enumOptions.size(); ++optionIndex) { if (definition.enumOptions[optionIndex].value == value.enumValue) { selectedIndex = static_cast(optionIndex); break; } } AppendStd140Int(buffer, selectedIndex); break; } case ShaderParameterType::Text: break; } } buffer.resize(AlignStd140(buffer.size(), 16), 0); glBindBuffer(GL_UNIFORM_BUFFER, mRenderer->mGlobalParamsUBO); if (mRenderer->mGlobalParamsUBOSize != static_cast(buffer.size())) { glBufferData(GL_UNIFORM_BUFFER, static_cast(buffer.size()), buffer.data(), GL_DYNAMIC_DRAW); mRenderer->mGlobalParamsUBOSize = static_cast(buffer.size()); } else { glBufferSubData(GL_UNIFORM_BUFFER, 0, static_cast(buffer.size()), buffer.data()); } glBindBufferBase(GL_UNIFORM_BUFFER, kGlobalParamsBindingPoint, mRenderer->mGlobalParamsUBO); glBindBuffer(GL_UNIFORM_BUFFER, 0); return true; } bool OpenGLComposite::CheckOpenGLExtensions() { mRenderer->mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable(); if (!mRenderer->mFastTransferExtensionAvailable) OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n"); return true; } ////////////////////////////////////////////