/* -LICENSE-START- ** Copyright (c) 2012 Blackmagic Design ** ** Permission is hereby granted, free of charge, to any person or organization ** obtaining a copy of the software and accompanying documentation (the ** "Software") to use, reproduce, display, distribute, sub-license, execute, ** and transmit the Software, and to prepare derivative works of the Software, ** and to permit third-parties to whom the Software is furnished to do so, in ** accordance with: ** ** (1) if the Software is obtained from Blackmagic Design, the End User License ** Agreement for the Software Development Kit ("EULA") available at ** https://www.blackmagicdesign.com/EULA/DeckLinkSDK; or ** ** (2) if the Software is obtained from any third party, such licensing terms ** as notified by that third party, ** ** and all subject to the following: ** ** (3) the copyright notices in the Software and this entire statement, ** including the above license grant, this restriction and the following ** disclaimer, must be included in all copies of the Software, in whole or in ** part, and all derivative works of the Software, unless such copies or ** derivative works are solely in the form of machine-executable object code ** generated by a source language processor. ** ** (4) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ** OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ** FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT ** SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE ** FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ** ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER ** DEALINGS IN THE SOFTWARE. ** ** A copy of the Software is available free of charge at ** https://www.blackmagicdesign.com/desktopvideo_sdk under the EULA. ** ** -LICENSE-END- */ #include "ControlServer.h" #include "DeckLinkDisplayMode.h" #include "DeckLinkFrameTransfer.h" #include "DeckLinkSession.h" #include "OpenGLComposite.h" #include "GLExtensions.h" #include "GlRenderConstants.h" #include "OpenGLRenderPass.h" #include "OpenGLShaderPrograms.h" #include "OscServer.h" #include "RuntimeControlBridge.h" #include #include #include OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) : hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC), mDeckLink(std::make_unique()), mRenderer(std::make_unique()) { InitializeCriticalSection(&pMutex); mRuntimeHost = std::make_unique(); mRenderPass = std::make_unique(*mRenderer); mShaderPrograms = std::make_unique(*mRenderer, *mRuntimeHost); mControlServer = std::make_unique(); mOscServer = std::make_unique(); } OpenGLComposite::~OpenGLComposite() { mDeckLink->ReleaseResources(); mRenderer->DestroyResources(); if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); DeleteCriticalSection(&pMutex); } bool OpenGLComposite::InitDeckLink() { BMDDisplayMode inputDisplayMode = bmdModeHD1080p5994; BMDDisplayMode outputDisplayMode = bmdModeHD1080p5994; std::string inputDisplayModeName = "1080p59.94"; std::string outputDisplayModeName = "1080p59.94"; std::string initFailureReason; if (mRuntimeHost && mRuntimeHost->GetRepoRoot().empty()) { std::string runtimeError; if (!mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } } if (mRuntimeHost) { if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetInputVideoFormat(), mRuntimeHost->GetInputFrameRate(), inputDisplayMode, inputDisplayModeName)) { const std::string error = "Unsupported DeckLink inputVideoFormat/inputFrameRate in config/runtime-host.json: " + mRuntimeHost->GetInputVideoFormat() + " / " + mRuntimeHost->GetInputFrameRate(); MessageBoxA(NULL, error.c_str(), "DeckLink input mode configuration error", MB_OK); return false; } if (!ResolveConfiguredDisplayMode(mRuntimeHost->GetOutputVideoFormat(), mRuntimeHost->GetOutputFrameRate(), outputDisplayMode, outputDisplayModeName)) { const std::string error = "Unsupported DeckLink outputVideoFormat/outputFrameRate in config/runtime-host.json: " + mRuntimeHost->GetOutputVideoFormat() + " / " + mRuntimeHost->GetOutputFrameRate(); MessageBoxA(NULL, error.c_str(), "DeckLink output mode configuration error", MB_OK); return false; } } if (!mDeckLink->DiscoverDevicesAndModes(inputDisplayMode, outputDisplayMode, inputDisplayModeName, outputDisplayModeName, initFailureReason)) { const char* title = initFailureReason == "Please install the Blackmagic DeckLink drivers to use the features of this application." ? "This application requires the DeckLink drivers installed." : "DeckLink initialization failed"; MessageBoxA(NULL, initFailureReason.c_str(), title, MB_OK | MB_ICONERROR); return false; } if (! CheckOpenGLExtensions()) { initFailureReason = "OpenGL extension checks failed."; goto error; } if (mDeckLink->inputFrameWidth != mDeckLink->outputFrameWidth || mDeckLink->inputFrameHeight != mDeckLink->outputFrameHeight) { mRenderer->mFastTransferExtensionAvailable = false; OutputDebugStringA("Input/output dimensions differ; using regular OpenGL transfer fallback instead of fast transfer.\n"); } if (! InitOpenGLState()) { initFailureReason = "OpenGL state initialization failed."; goto error; } PublishDeckLinkOutputStatus(mDeckLink->outputModelName.empty() ? "DeckLink output device selected." : ("Selected output device: " + mDeckLink->outputModelName)); // Resize window to match output video frame, but scale large formats down by half for viewing. if (mDeckLink->outputFrameWidth < 1920) resizeWindow(mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight); else resizeWindow(mDeckLink->outputFrameWidth / 2, mDeckLink->outputFrameHeight / 2); if (mRenderer->mFastTransferExtensionAvailable) { // Initialize fast video frame transfers if (! VideoFrameTransfer::initialize(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mRenderer->mCaptureTexture, mRenderer->mOutputTexture)) { MessageBox(NULL, _T("Cannot initialize video transfers."), _T("VideoFrameTransfer error."), MB_OK); goto error; } } if (!mDeckLink->ConfigureInput(this, hGLDC, hGLRC, inputDisplayMode, initFailureReason)) { goto error; } if (!mDeckLink->input && mRuntimeHost) { mRuntimeHost->SetSignalStatus(false, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->inputDisplayModeName); } if (!mDeckLink->ConfigureOutput(this, hGLDC, hGLRC, outputDisplayMode, mRuntimeHost && mRuntimeHost->ExternalKeyingEnabled(), initFailureReason)) { goto error; } PublishDeckLinkOutputStatus(mDeckLink->statusMessage); return true; error: if (!initFailureReason.empty()) MessageBoxA(NULL, initFailureReason.c_str(), "DeckLink initialization failed", MB_OK | MB_ICONERROR); mDeckLink->ReleaseResources(); return false; } void OpenGLComposite::paintGL() { if (!TryEnterCriticalSection(&pMutex)) { ValidateRect(hGLWnd, NULL); return; } mRenderer->PresentToWindow(hGLDC, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight); ValidateRect(hGLWnd, NULL); LeaveCriticalSection(&pMutex); } void OpenGLComposite::resizeGL(WORD width, WORD height) { // We don't set the project or model matrices here since the window data is copied directly from // an off-screen FBO in paintGL(). Just save the width and height for use in paintGL(). mRenderer->ResizeView(width, height); } void OpenGLComposite::resizeWindow(int width, int height) { RECT r; if (GetWindowRect(hGLWnd, &r)) { SetWindowPos(hGLWnd, HWND_TOP, r.left, r.top, r.left + width, r.top + height, 0); } } void OpenGLComposite::PublishDeckLinkOutputStatus(const std::string& statusMessage) { if (!mRuntimeHost) return; if (!statusMessage.empty()) mDeckLink->statusMessage = statusMessage; mRuntimeHost->SetDeckLinkOutputStatus( mDeckLink->outputModelName, mDeckLink->supportsInternalKeying, mDeckLink->supportsExternalKeying, mDeckLink->keyerInterfaceAvailable, mRuntimeHost->ExternalKeyingEnabled(), mDeckLink->externalKeyingActive, mDeckLink->statusMessage); } bool OpenGLComposite::InitOpenGLState() { if (! ResolveGLExtensions()) return false; std::string runtimeError; if (mRuntimeHost->GetRepoRoot().empty() && !mRuntimeHost->Initialize(runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime host failed to initialize", MB_OK); return false; } if (!StartRuntimeControlServices(*this, *mRuntimeHost, *mControlServer, *mOscServer, runtimeError)) { MessageBoxA(NULL, runtimeError.c_str(), "Runtime control services failed to start", MB_OK); return false; } // Prepare the runtime shader program generated from the active shader package. char compilerErrorMessage[1024]; if (!mShaderPrograms->CompileDecodeShader(sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL decode shader failed to load or compile", MB_OK); return false; } if (!mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage)) { MessageBoxA(NULL, compilerErrorMessage, "OpenGL shader failed to load or compile", MB_OK); return false; } mShaderPrograms->ResetTemporalHistoryState(); std::string rendererError; if (!mRenderer->InitializeResources(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, rendererError)) { MessageBoxA(NULL, rendererError.c_str(), "OpenGL initialization error.", MB_OK); return false; } broadcastRuntimeState(); return true; } // // Update the captured video frame texture // void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource) { mDeckLink->hasNoInputSource = hasNoInputSource; if (mRuntimeHost) mRuntimeHost->SetSignalStatus(!hasNoInputSource, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, mDeckLink->inputDisplayModeName); if (mDeckLink->hasNoInputSource) return; // don't transfer texture when there's no input long textureSize = inputFrame->GetRowBytes() * inputFrame->GetHeight(); IDeckLinkVideoBuffer* inputFrameBuffer = NULL; void* videoPixels; if (inputFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&inputFrameBuffer) != S_OK) return; if (inputFrameBuffer->StartAccess(bmdBufferAccessRead) != S_OK) { inputFrameBuffer->Release(); return; } inputFrameBuffer->GetBytes(&videoPixels); EnterCriticalSection(&pMutex); wglMakeCurrent( hGLDC, hGLRC ); // make OpenGL context current in this thread if (mRenderer->mFastTransferExtensionAvailable) { CComQIPtr allocator(inputFrameBuffer); if (!allocator || !allocator->transferFrame(videoPixels, mRenderer->mCaptureTexture)) OutputDebugStringA("Capture: transferFrame() failed\n"); allocator->waitForTransferComplete(videoPixels); } else { // Use a straightforward texture buffer glBindBuffer(GL_PIXEL_UNPACK_BUFFER, mRenderer->mUnpinnedTextureBuffer); glBufferData(GL_PIXEL_UNPACK_BUFFER, textureSize, videoPixels, GL_DYNAMIC_DRAW); glBindTexture(GL_TEXTURE_2D, mRenderer->mCaptureTexture); // NULL for last arg indicates use current GL_PIXEL_UNPACK_BUFFER target as texture data glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, mDeckLink->inputFrameWidth / 2, mDeckLink->inputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL); glBindTexture(GL_TEXTURE_2D, 0); glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); } wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); inputFrameBuffer->EndAccess(bmdBufferAccessRead); inputFrameBuffer->Release(); } // Render the live video texture through the runtime shader into the off-screen framebuffer. // Read the result back from the frame buffer and schedule it for playout. void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult) { EnterCriticalSection(&pMutex); // Get the first frame from the queue IDeckLinkMutableVideoFrame* outputVideoFrame = mDeckLink->outputVideoFrameQueue.front(); mDeckLink->outputVideoFrameQueue.push_back(outputVideoFrame); mDeckLink->outputVideoFrameQueue.pop_front(); // make GL context current in this thread wglMakeCurrent( hGLDC, hGLRC ); // Draw the effect output to the off-screen framebuffer. const auto renderStartTime = std::chrono::steady_clock::now(); if (mRenderer->mFastTransferExtensionAvailable) VideoFrameTransfer::beginTextureInUse(VideoFrameTransfer::GPUtoCPU); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mIdFrameBuf); renderEffect(); glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mIdFrameBuf); glBindFramebuffer(GL_DRAW_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glBlitFramebuffer(0, 0, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, 0, 0, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, GL_COLOR_BUFFER_BIT, GL_LINEAR); glBindFramebuffer(GL_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glFlush(); if (mRenderer->mFastTransferExtensionAvailable) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::GPUtoCPU); const auto renderEndTime = std::chrono::steady_clock::now(); if (mRuntimeHost) { const double frameBudgetMilliseconds = mDeckLink->frameTimescale != 0 ? (static_cast(mDeckLink->frameDuration) * 1000.0) / static_cast(mDeckLink->frameTimescale) : 0.0; const double renderMilliseconds = std::chrono::duration_cast>(renderEndTime - renderStartTime).count(); mRuntimeHost->SetPerformanceStats(frameBudgetMilliseconds, renderMilliseconds); } if (mRuntimeHost) mRuntimeHost->AdvanceFrame(); IDeckLinkVideoBuffer* outputVideoFrameBuffer; if (outputVideoFrame->QueryInterface(IID_IDeckLinkVideoBuffer, (void**)&outputVideoFrameBuffer) != S_OK) { LeaveCriticalSection(&pMutex); return; } if (outputVideoFrameBuffer->StartAccess(bmdBufferAccessWrite) != S_OK) { outputVideoFrameBuffer->Release(); LeaveCriticalSection(&pMutex); return; } void* pFrame; outputVideoFrameBuffer->GetBytes(&pFrame); if (mRenderer->mFastTransferExtensionAvailable) { // Finished with mRenderer->mCaptureTexture if (!mDeckLink->hasNoInputSource) VideoFrameTransfer::endTextureInUse(VideoFrameTransfer::CPUtoGPU); if (! mDeckLink->playoutAllocator->transferFrame(pFrame, mRenderer->mOutputTexture)) OutputDebugStringA("Playback: transferFrame() failed\n"); paintGL(); // Wait for transfer to system memory to complete mDeckLink->playoutAllocator->waitForTransferComplete(pFrame); } else { glBindFramebuffer(GL_READ_FRAMEBUFFER, mRenderer->mOutputFrameBuf); glReadPixels(0, 0, mDeckLink->outputFrameWidth, mDeckLink->outputFrameHeight, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, pFrame); paintGL(); } outputVideoFrameBuffer->EndAccess(bmdBufferAccessWrite); outputVideoFrameBuffer->Release(); // If the last completed frame was late or dropped, bump the scheduled time further into the future if (completionResult == bmdOutputFrameDisplayedLate || completionResult == bmdOutputFrameDropped) mDeckLink->totalPlayoutFrames += 2; // Schedule the next frame for playout HRESULT hr = mDeckLink->output->ScheduleVideoFrame(outputVideoFrame, (mDeckLink->totalPlayoutFrames * mDeckLink->frameDuration), mDeckLink->frameDuration, mDeckLink->frameTimescale); if (SUCCEEDED(hr)) mDeckLink->totalPlayoutFrames++; wglMakeCurrent( NULL, NULL ); LeaveCriticalSection(&pMutex); } bool OpenGLComposite::Start() { return mDeckLink->Start(mDeckLink->outputFrameHeight); } bool OpenGLComposite::Stop() { if (mOscServer) mOscServer->Stop(); if (mControlServer) mControlServer->Stop(); const bool wasExternalKeyingActive = mDeckLink->externalKeyingActive; mDeckLink->Stop(); if (wasExternalKeyingActive) PublishDeckLinkOutputStatus("External keying has been disabled."); return true; } bool OpenGLComposite::ReloadShader() { char compilerErrorMessage[1024]; EnterCriticalSection(&pMutex); wglMakeCurrent(hGLDC, hGLRC); bool success = mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage); if (mRuntimeHost) mRuntimeHost->ClearReloadRequest(); wglMakeCurrent(NULL, NULL); LeaveCriticalSection(&pMutex); if (!success) { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); MessageBoxA(NULL, compilerErrorMessage, "Slang shader reload failed", MB_OK); } else { if (mRuntimeHost) mRuntimeHost->SetCompileStatus(true, "Shader compiled successfully."); broadcastRuntimeState(); } return success; } void OpenGLComposite::renderEffect() { PollRuntimeChanges(); const bool hasInputSource = !mDeckLink->hasNoInputSource; const std::vector layerStates = mRuntimeHost ? mRuntimeHost->GetLayerRenderStates(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight) : std::vector(); const unsigned historyCap = mRuntimeHost ? mRuntimeHost->GetMaxTemporalHistoryFrames() : 0; mRenderPass->Render( hasInputSource, layerStates, mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, historyCap, [this](const RuntimeRenderState& state, LayerProgram::TextBinding& textBinding, std::string& error) { return mShaderPrograms->UpdateTextBindingTexture(state, textBinding, error); }, [this](const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength) { return mShaderPrograms->UpdateGlobalParamsBuffer(state, availableSourceHistoryLength, availableTemporalHistoryLength); }); } bool OpenGLComposite::PollRuntimeChanges() { if (!mRuntimeHost) return true; bool registryChanged = false; bool reloadRequested = false; std::string runtimeError; if (!mRuntimeHost->PollFileChanges(registryChanged, reloadRequested, runtimeError)) { mRuntimeHost->SetCompileStatus(false, runtimeError); broadcastRuntimeState(); return false; } if (registryChanged) broadcastRuntimeState(); if (!reloadRequested) return true; char compilerErrorMessage[1024] = {}; if (!mShaderPrograms->CompileLayerPrograms(mDeckLink->inputFrameWidth, mDeckLink->inputFrameHeight, sizeof(compilerErrorMessage), compilerErrorMessage)) { mRuntimeHost->SetCompileStatus(false, compilerErrorMessage); mRuntimeHost->ClearReloadRequest(); broadcastRuntimeState(); return false; } mShaderPrograms->ResetTemporalHistoryState(); broadcastRuntimeState(); return true; } void OpenGLComposite::broadcastRuntimeState() { if (mControlServer) mControlServer->BroadcastState(); } void OpenGLComposite::resetTemporalHistoryState() { mShaderPrograms->ResetTemporalHistoryState(); } bool OpenGLComposite::CheckOpenGLExtensions() { mRenderer->mFastTransferExtensionAvailable = VideoFrameTransfer::checkFastMemoryTransferAvailable(); if (!mRenderer->mFastTransferExtensionAvailable) OutputDebugStringA("Fast memory transfer extension not available, using regular OpenGL transfer fallback instead\n"); return true; } ////////////////////////////////////////////