Initial audio support

apps/LoopThroughWithOpenGLCompositing/AudioSupport.cpp

@@ -0,0 +1,206 @@
+#include "AudioSupport.h"
+
+#include <algorithm>
+#include <cmath>
+#include <iterator>
+#include <limits>
+
+namespace
+{
+constexpr float kInt32ToFloat = 1.0f / 2147483648.0f;
+constexpr std::size_t kAnalysisWindowSamples = 1024;
+constexpr std::size_t kMaxBufferedAudioFrames = kAudioSampleRate * 10;
+
+float Clamp01(float value)
+{
+	return std::max(0.0f, std::min(1.0f, value));
+}
+
+float SampleToFloat(int32_t sample)
+{
+	return std::max(-1.0f, std::min(1.0f, static_cast<float>(sample) * kInt32ToFloat));
+}
+
+float GoertzelMagnitude(const std::vector<float>& samples, float frequency)
+{
+	if (samples.empty())
+		return 0.0f;
+
+	const double omega = 2.0 * 3.14159265358979323846 * static_cast<double>(frequency) / static_cast<double>(kAudioSampleRate);
+	const double coefficient = 2.0 * std::cos(omega);
+	double q0 = 0.0;
+	double q1 = 0.0;
+	double q2 = 0.0;
+
+	for (float sample : samples)
+	{
+		q0 = coefficient * q1 - q2 + static_cast<double>(sample);
+		q2 = q1;
+		q1 = q0;
+	}
+
+	const double power = q1 * q1 + q2 * q2 - coefficient * q1 * q2;
+	return static_cast<float>(std::sqrt(std::max(0.0, power)) / static_cast<double>(samples.size()));
+}
+}
+
+uint64_t AudioSampleTimeForVideoFrame(uint64_t videoFrameIndex, uint64_t frameDuration, uint64_t frameTimescale, uint64_t audioSampleRate)
+{
+	if (frameTimescale == 0)
+		return 0;
+
+	const uint64_t numerator = videoFrameIndex * frameDuration * audioSampleRate;
+	return (numerator + frameTimescale / 2) / frameTimescale;
+}
+
+unsigned AudioSamplesForVideoFrame(uint64_t videoFrameIndex, uint64_t frameDuration, uint64_t frameTimescale, uint64_t audioSampleRate)
+{
+	const uint64_t start = AudioSampleTimeForVideoFrame(videoFrameIndex, frameDuration, frameTimescale, audioSampleRate);
+	const uint64_t end = AudioSampleTimeForVideoFrame(videoFrameIndex + 1, frameDuration, frameTimescale, audioSampleRate);
+	return static_cast<unsigned>(end > start ? end - start : 0);
+}
+
+void AudioDelayBuffer::Reset(unsigned delaySampleFrames)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mSamples.clear();
+	mSamples.resize(static_cast<std::size_t>(delaySampleFrames) * kAudioChannelCount, 0);
+	mUnderrunCount = 0;
+}
+
+void AudioDelayBuffer::PushInterleaved(const int32_t* samples, std::size_t sampleFrameCount)
+{
+	if (!samples || sampleFrameCount == 0)
+		return;
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	const std::size_t sampleCount = sampleFrameCount * kAudioChannelCount;
+	for (std::size_t index = 0; index < sampleCount; ++index)
+		mSamples.push_back(samples[index]);
+
+	const std::size_t maxSamples = kMaxBufferedAudioFrames * kAudioChannelCount;
+	while (mSamples.size() > maxSamples)
+		mSamples.pop_front();
+}
+
+AudioFrameBlock AudioDelayBuffer::Pop(std::size_t sampleFrameCount, bool& underrun)
+{
+	AudioFrameBlock block;
+	block.interleavedSamples.resize(sampleFrameCount * kAudioChannelCount, 0);
+
+	std::lock_guard<std::mutex> lock(mMutex);
+	const std::size_t requestedSamples = sampleFrameCount * kAudioChannelCount;
+	underrun = mSamples.size() < requestedSamples;
+	if (underrun)
+		++mUnderrunCount;
+
+	const std::size_t availableSamples = std::min(requestedSamples, mSamples.size());
+	for (std::size_t index = 0; index < availableSamples; ++index)
+	{
+		block.interleavedSamples[index] = mSamples.front();
+		mSamples.pop_front();
+	}
+
+	return block;
+}
+
+unsigned AudioDelayBuffer::BufferedSampleFrames() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return static_cast<unsigned>(mSamples.size() / kAudioChannelCount);
+}
+
+uint64_t AudioDelayBuffer::UnderrunCount() const
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	return mUnderrunCount;
+}
+
+void AudioAnalyzer::Reset()
+{
+	mMonoHistory.clear();
+	mSmoothedBands = { 0.0f, 0.0f, 0.0f, 0.0f };
+	mCurrent = AudioAnalysisSnapshot();
+}
+
+AudioAnalysisSnapshot AudioAnalyzer::Analyze(const AudioFrameBlock& block)
+{
+	AudioAnalysisSnapshot next;
+	double sumSquares[2] = { 0.0, 0.0 };
+	float peak[2] = { 0.0f, 0.0f };
+	double monoSumSquares = 0.0;
+	float monoPeak = 0.0f;
+	const std::size_t frames = block.frameCount();
+
+	for (std::size_t frame = 0; frame < frames; ++frame)
+	{
+		const float left = SampleToFloat(block.interleavedSamples[frame * 2]);
+		const float right = SampleToFloat(block.interleavedSamples[frame * 2 + 1]);
+		const float mono = (left + right) * 0.5f;
+
+		sumSquares[0] += static_cast<double>(left) * left;
+		sumSquares[1] += static_cast<double>(right) * right;
+		peak[0] = std::max(peak[0], std::abs(left));
+		peak[1] = std::max(peak[1], std::abs(right));
+		monoSumSquares += static_cast<double>(mono) * mono;
+		monoPeak = std::max(monoPeak, std::abs(mono));
+
+		mMonoHistory.push_back(mono);
+		while (mMonoHistory.size() > kAnalysisWindowSamples)
+			mMonoHistory.pop_front();
+	}
+
+	if (frames > 0)
+	{
+		next.rms[0] = static_cast<float>(std::sqrt(sumSquares[0] / static_cast<double>(frames)));
+		next.rms[1] = static_cast<float>(std::sqrt(sumSquares[1] / static_cast<double>(frames)));
+		next.peak[0] = peak[0];
+		next.peak[1] = peak[1];
+		next.monoRms = static_cast<float>(std::sqrt(monoSumSquares / static_cast<double>(frames)));
+		next.monoPeak = monoPeak;
+	}
+
+	std::vector<float> window(mMonoHistory.begin(), mMonoHistory.end());
+	const float bandFrequencies[4] = { 90.0f, 300.0f, 1200.0f, 5000.0f };
+	for (std::size_t band = 0; band < next.bands.size(); ++band)
+	{
+		const float raw = Clamp01(GoertzelMagnitude(window, bandFrequencies[band]) * 8.0f);
+		const float smoothing = raw > mSmoothedBands[band] ? 0.45f : 0.12f;
+		mSmoothedBands[band] = mSmoothedBands[band] + (raw - mSmoothedBands[band]) * smoothing;
+		next.bands[band] = Clamp01(mSmoothedBands[band]);
+	}
+
+	for (unsigned x = 0; x < kAudioTextureWidth; ++x)
+	{
+		float mono = 0.0f;
+		if (!mMonoHistory.empty())
+		{
+			const std::size_t historyIndex = static_cast<std::size_t>(
+				(static_cast<uint64_t>(x) * static_cast<uint64_t>(mMonoHistory.size())) / kAudioTextureWidth);
+			auto it = mMonoHistory.begin();
+			std::advance(it, std::min(historyIndex, mMonoHistory.size() - 1));
+			mono = *it;
+		}
+
+		const std::size_t waveformOffset = x * 4;
+		next.texture[waveformOffset + 0] = mono * 0.5f + 0.5f;
+		next.texture[waveformOffset + 1] = next.texture[waveformOffset + 0];
+		next.texture[waveformOffset + 2] = next.monoRms;
+		next.texture[waveformOffset + 3] = 1.0f;
+
+		const float bandPosition = static_cast<float>(x) / static_cast<float>(kAudioTextureWidth - 1);
+		const float scaled = bandPosition * static_cast<float>(next.bands.size() - 1);
+		const unsigned bandA = static_cast<unsigned>(std::floor(scaled));
+		const unsigned bandB = std::min<unsigned>(bandA + 1, static_cast<unsigned>(next.bands.size() - 1));
+		const float t = scaled - static_cast<float>(bandA);
+		const float spectrum = next.bands[bandA] * (1.0f - t) + next.bands[bandB] * t;
+		const std::size_t spectrumOffset = (kAudioTextureWidth + x) * 4;
+		next.texture[spectrumOffset + 0] = spectrum;
+		next.texture[spectrumOffset + 1] = next.bands[0];
+		next.texture[spectrumOffset + 2] = next.bands[1];
+		next.texture[spectrumOffset + 3] = next.bands[2];
+	}
+
+	mCurrent = next;
+	return mCurrent;
+}

apps/LoopThroughWithOpenGLCompositing/AudioSupport.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <deque>
+#include <mutex>
+#include <vector>
+
+constexpr unsigned kAudioSampleRate = 48000;
+constexpr unsigned kAudioChannelCount = 2;
+constexpr unsigned kAudioTextureWidth = 64;
+constexpr unsigned kAudioTextureHeight = 2;
+
+struct AudioFrameBlock
+{
+	std::vector<int32_t> interleavedSamples;
+
+	std::size_t frameCount() const
+	{
+		return interleavedSamples.size() / kAudioChannelCount;
+	}
+};
+
+struct AudioAnalysisSnapshot
+{
+	std::array<float, 2> rms = { 0.0f, 0.0f };
+	std::array<float, 2> peak = { 0.0f, 0.0f };
+	float monoRms = 0.0f;
+	float monoPeak = 0.0f;
+	std::array<float, 4> bands = { 0.0f, 0.0f, 0.0f, 0.0f };
+	std::array<float, kAudioTextureWidth * kAudioTextureHeight * 4> texture = {};
+};
+
+struct AudioStatusSnapshot
+{
+	bool enabled = false;
+	unsigned bufferedSampleFrames = 0;
+	uint64_t underrunCount = 0;
+	AudioAnalysisSnapshot analysis;
+};
+
+class AudioDelayBuffer
+{
+public:
+	void Reset(unsigned delaySampleFrames);
+	void PushInterleaved(const int32_t* samples, std::size_t sampleFrameCount);
+	AudioFrameBlock Pop(std::size_t sampleFrameCount, bool& underrun);
+	unsigned BufferedSampleFrames() const;
+	uint64_t UnderrunCount() const;
+
+private:
+	mutable std::mutex mMutex;
+	std::deque<int32_t> mSamples;
+	uint64_t mUnderrunCount = 0;
+};
+
+class AudioAnalyzer
+{
+public:
+	void Reset();
+	AudioAnalysisSnapshot Analyze(const AudioFrameBlock& block);
+	const AudioAnalysisSnapshot& Current() const { return mCurrent; }
+
+private:
+	std::deque<float> mMonoHistory;
+	std::array<float, 4> mSmoothedBands = { 0.0f, 0.0f, 0.0f, 0.0f };
+	AudioAnalysisSnapshot mCurrent;
+};
+
+uint64_t AudioSampleTimeForVideoFrame(uint64_t videoFrameIndex, uint64_t frameDuration, uint64_t frameTimescale, uint64_t audioSampleRate = kAudioSampleRate);
+unsigned AudioSamplesForVideoFrame(uint64_t videoFrameIndex, uint64_t frameDuration, uint64_t frameTimescale, uint64_t audioSampleRate = kAudioSampleRate);

apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.cpp

@@ -60,9 +60,16 @@ DEFINE_GUID(IID_PinnedMemoryAllocator,
 namespace
 {
 constexpr GLuint kDecodedVideoTextureUnit = 1;
-constexpr GLuint kSourceHistoryTextureUnitBase = 2;
+constexpr GLuint kAudioDataTextureUnit = 2;
+constexpr GLuint kSourceHistoryTextureUnitBase = 3;
 constexpr GLuint kPackedVideoTextureUnit = 2;
 constexpr GLuint kGlobalParamsBindingPoint = 0;
+constexpr unsigned kVideoPrerollFrameCount = 5;
+constexpr unsigned kAudioOutputWaterLevelSampleFrames = kAudioSampleRate / 2;
+
+#ifndef GL_RGBA32F
+#define GL_RGBA32F 0x8814
+#endif
 const char* kVertexShaderSource =
 	"#version 430 core\n"
 	"out vec2 vTexCoord;\n"
@@ -315,8 +322,10 @@ void AppendStd140Vec4(std::vector<unsigned char>& buffer, float x, float y, floa
 OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
 	hGLWnd(hWnd), hGLDC(hDC), hGLRC(hRC),
 	mCaptureDelegate(NULL), mPlayoutDelegate(NULL),
-	mDLInput(NULL), mDLOutput(NULL), mDLKeyer(NULL),
+	mDLInput(NULL), mDLOutput(NULL), mDLInputConfiguration(NULL), mDLKeyer(NULL),
 	mPlayoutAllocator(NULL),
+	mTotalPlayoutFrames(0),
+	mNextAudioSampleFrame(0),
 	mInputFrameWidth(0), mInputFrameHeight(0),
 	mOutputFrameWidth(0), mOutputFrameHeight(0),
 	mInputDisplayModeName("1080p59.94"),
@@ -332,6 +341,7 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
 	mLayerTempTexture(0),
 	mFBOTexture(0),
 	mOutputTexture(0),
+	mAudioDataTexture(0),
 	mUnpinnedTextureBuffer(0),
 	mDecodeFrameBuf(0),
 	mLayerTempFrameBuf(0),
@@ -347,6 +357,8 @@ OpenGLComposite::OpenGLComposite(HWND hWnd, HDC hDC, HGLRC hRC) :
 	mGlobalParamsUBOSize(0),
 	mViewWidth(0),
 	mViewHeight(0),
+	mAudioEnabled(false),
+	mAudioPrerolling(false),
 	mTemporalHistoryNeedsReset(true)
 {
 	InitializeCriticalSection(&pMutex);
@@ -362,6 +374,12 @@ OpenGLComposite::~OpenGLComposite()
 	{
 		mDLInput->SetCallback(NULL);
 
+		if (mDLInputConfiguration != NULL)
+		{
+			mDLInputConfiguration->Release();
+			mDLInputConfiguration = NULL;
+		}
+
 		mDLInput->Release();
 		mDLInput = NULL;
 	}
@@ -394,6 +412,7 @@ OpenGLComposite::~OpenGLComposite()
 		}
 
 		mDLOutput->SetScheduledFrameCompletionCallback(NULL);
+		mDLOutput->SetAudioCallback(NULL);
 
 		mDLOutput->Release();
 		mDLOutput = NULL;
@@ -435,6 +454,8 @@ OpenGLComposite::~OpenGLComposite()
 		glDeleteTextures(1, &mFBOTexture);
 	if (mOutputTexture != 0)
 		glDeleteTextures(1, &mOutputTexture);
+	if (mAudioDataTexture != 0)
+		glDeleteTextures(1, &mAudioDataTexture);
 	if (mOutputFrameBuf != 0)
 		glDeleteFramebuffers(1, &mOutputFrameBuf);
 	if (mUnpinnedTextureBuffer != 0)
@@ -667,6 +688,26 @@ bool OpenGLComposite::InitDeckLink()
 			goto error;
 	}
 
+	mAudioEnabled = mRuntimeHost ? mRuntimeHost->AudioEnabled() : true;
+	if (mAudioEnabled)
+	{
+		if (mDLInput->QueryInterface(IID_IDeckLinkConfiguration, (void**)&mDLInputConfiguration) == S_OK && mDLInputConfiguration != NULL)
+		{
+			if (mDLInputConfiguration->SetInt(bmdDeckLinkConfigAudioInputConnection, bmdAudioConnectionEmbedded) != S_OK)
+				OutputDebugStringA("Could not force DeckLink audio input connection to embedded; using current device setting.\n");
+		}
+		else
+		{
+			OutputDebugStringA("Could not query DeckLink input configuration; using current audio input connection.\n");
+		}
+	}
+
+	if (mAudioEnabled && mDLInput->EnableAudioInput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, kAudioChannelCount) != S_OK)
+	{
+		OutputDebugStringA("Could not enable DeckLink audio input; continuing without audio.\n");
+		mAudioEnabled = false;
+	}
+
 	mCaptureDelegate = new CaptureDelegate(this);
 	if (mDLInput->SetCallback(mCaptureDelegate) != S_OK)
 		goto error;
@@ -680,6 +721,13 @@ bool OpenGLComposite::InitDeckLink()
 	if (mDLOutput->EnableVideoOutput(outputDisplayMode, bmdVideoOutputFlagDefault) != S_OK)
 		goto error;
 
+	if (mAudioEnabled && mDLOutput->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, kAudioChannelCount, bmdAudioOutputStreamTimestamped) != S_OK)
+	{
+		OutputDebugStringA("Could not enable DeckLink audio output; continuing without audio.\n");
+		mDLInput->DisableAudioInput();
+		mAudioEnabled = false;
+	}
+
 	if (mDLOutput->QueryInterface(IID_IDeckLinkKeyer, (void**)&mDLKeyer) == S_OK && mDLKeyer != NULL)
 		mDeckLinkKeyerInterfaceAvailable = true;
 
@@ -748,6 +796,14 @@ bool OpenGLComposite::InitDeckLink()
 	if (mDLOutput->SetScheduledFrameCompletionCallback(mPlayoutDelegate) != S_OK)
 		goto error;
 
+	if (mAudioEnabled && mDLOutput->SetAudioCallback(mPlayoutDelegate) != S_OK)
+	{
+		OutputDebugStringA("Could not set DeckLink audio output callback; continuing without audio.\n");
+		mDLInput->DisableAudioInput();
+		mDLOutput->DisableAudioOutput();
+		mAudioEnabled = false;
+	}
+
 	bSuccess = true;
 
 error:
@@ -770,6 +826,11 @@ error:
 			mDLOutput->Release();
 			mDLOutput = NULL;
 		}
+		if (mDLInputConfiguration != NULL)
+		{
+			mDLInputConfiguration->Release();
+			mDLInputConfiguration = NULL;
+		}
 	}
 
 	if (pDL != NULL)
@@ -1052,6 +1113,14 @@ bool OpenGLComposite::InitOpenGLState()
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 	glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, mOutputFrameWidth, mOutputFrameHeight, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, NULL);
 
+	glGenTextures(1, &mAudioDataTexture);
+	glBindTexture(GL_TEXTURE_2D, mAudioDataTexture);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+	glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, kAudioTextureWidth, kAudioTextureHeight, 0, GL_RGBA, GL_FLOAT, mAudioAnalysis.texture.data());
+
 	glBindFramebuffer(GL_FRAMEBUFFER, mOutputFrameBuf);
 	glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mOutputTexture, 0);
 	glStatus = glCheckFramebufferStatus(GL_FRAMEBUFFER);
@@ -1135,6 +1204,73 @@ void OpenGLComposite::VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bo
 	inputFrameBuffer->Release();
 }
 
+void OpenGLComposite::AudioPacketArrived(IDeckLinkAudioInputPacket* audioPacket)
+{
+	if (!mAudioEnabled || !audioPacket)
+		return;
+
+	void* audioBytes = nullptr;
+	if (audioPacket->GetBytes(&audioBytes) != S_OK || !audioBytes)
+		return;
+
+	const long sampleFrameCount = audioPacket->GetSampleFrameCount();
+	if (sampleFrameCount <= 0)
+		return;
+
+	mAudioDelayBuffer.PushInterleaved(static_cast<const int32_t*>(audioBytes), static_cast<std::size_t>(sampleFrameCount));
+	updateAudioStatus();
+}
+
+HRESULT OpenGLComposite::RenderAudioSamples(BOOL preroll)
+{
+	if (!mAudioEnabled || !mDLOutput)
+		return S_OK;
+
+	std::lock_guard<std::mutex> audioLock(mAudioStateMutex);
+
+	unsigned bufferedSampleFrames = 0;
+	if (mDLOutput->GetBufferedAudioSampleFrameCount(&bufferedSampleFrames) != S_OK)
+	{
+		OutputDebugStringA("Could not query DeckLink buffered audio sample count.\n");
+		return E_FAIL;
+	}
+
+	const unsigned delayedSampleFrames = delayedAudioSampleFrames();
+	const unsigned waterLevel = kAudioOutputWaterLevelSampleFrames > delayedSampleFrames
+		? kAudioOutputWaterLevelSampleFrames
+		: delayedSampleFrames;
+	if (bufferedSampleFrames >= waterLevel)
+		return S_OK;
+
+	const unsigned requestedSampleFrames = waterLevel - bufferedSampleFrames;
+	bool underrun = false;
+	AudioFrameBlock audioBlock = mAudioDelayBuffer.Pop(requestedSampleFrames, underrun);
+	mAudioAnalysis = mAudioAnalyzer.Analyze(audioBlock);
+
+	unsigned sampleFramesWritten = 0;
+	const unsigned sampleFrames = static_cast<unsigned>(audioBlock.frameCount());
+	const HRESULT scheduleResult = mDLOutput->ScheduleAudioSamples(
+		audioBlock.interleavedSamples.data(),
+		sampleFrames,
+		static_cast<BMDTimeValue>(mNextAudioSampleFrame),
+		kAudioSampleRate,
+		&sampleFramesWritten);
+
+	if (scheduleResult == S_OK)
+	{
+		if (sampleFramesWritten == 0 && sampleFrames > 0)
+			OutputDebugStringA("DeckLink accepted audio schedule call but wrote 0 sample frames.\n");
+		mNextAudioSampleFrame += sampleFramesWritten;
+	}
+	else
+	{
+		OutputDebugStringA("DeckLink ScheduleAudioSamples failed while topping up audio output.\n");
+	}
+
+	updateAudioStatus();
+	return scheduleResult;
+}
+
 // Render the live video texture through the runtime shader into the off-screen framebuffer.
 // Read the result back from the frame buffer and schedule it for playout.
 void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completionResult)
@@ -1149,6 +1285,16 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
 	// make GL context current in this thread
 	wglMakeCurrent( hGLDC, hGLRC );
 
+	if (mAudioEnabled)
+	{
+		AudioAnalysisSnapshot audioAnalysis;
+		{
+			std::lock_guard<std::mutex> audioLock(mAudioStateMutex);
+			audioAnalysis = mAudioAnalysis;
+		}
+		updateAudioDataTexture(audioAnalysis);
+	}
+
 	// Draw the effect output to the off-screen framebuffer.
 	const auto renderStartTime = std::chrono::steady_clock::now();
 	if (mFastTransferExtensionAvailable)
@@ -1231,9 +1377,25 @@ void OpenGLComposite::PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame,
 bool OpenGLComposite::Start()
 {
 	mTotalPlayoutFrames = 0;
+	initializeAudioDelay();
+	if (mAudioEnabled)
+	{
+		mDLOutput->FlushBufferedAudioSamples();
+		if (mDLOutput->BeginAudioPreroll() != S_OK)
+		{
+			OutputDebugStringA("Could not begin DeckLink audio preroll; continuing without audio.\n");
+			mDLInput->DisableAudioInput();
+			mDLOutput->DisableAudioOutput();
+			mAudioEnabled = false;
+		}
+		else
+		{
+			mAudioPrerolling = true;
+		}
+	}
 
 	// Preroll frames
-	for (unsigned i = 0; i < 5; i++)
+	for (unsigned i = 0; i < kVideoPrerollFrameCount; i++)
 	{
 		// Take each video frame from the front of the queue and move it to the back
 		IDeckLinkMutableVideoFrame* outputVideoFrame = mDLOutputVideoFrameQueue.front();
@@ -1264,8 +1426,26 @@ bool OpenGLComposite::Start()
 		mTotalPlayoutFrames++;
 	}
 
-	mDLInput->StartStreams();
-	mDLOutput->StartScheduledPlayback(0, mFrameTimescale, 1.0);
+	if (mAudioEnabled)
+		RenderAudioSamples(TRUE);
+
+	if (mAudioPrerolling)
+	{
+		if (mDLOutput->EndAudioPreroll() != S_OK)
+		{
+			OutputDebugStringA("Could not end DeckLink audio preroll; continuing without audio.\n");
+			mDLInput->DisableAudioInput();
+			mDLOutput->DisableAudioOutput();
+			mAudioEnabled = false;
+		}
+		mAudioPrerolling = false;
+	}
+
+	if (mDLInput->StartStreams() != S_OK)
+		return false;
+
+	if (mDLOutput->StartScheduledPlayback(0, mFrameTimescale, 1.0) != S_OK)
+		return false;
 
 	return true;
 }
@@ -1297,9 +1477,16 @@ bool OpenGLComposite::Stop()
 
 	mDLInput->StopStreams();
 	mDLInput->DisableVideoInput();
+	if (mAudioEnabled)
+		mDLInput->DisableAudioInput();
 
 	mDLOutput->StopScheduledPlayback(0, NULL, 0);
+	mDLOutput->SetAudioCallback(NULL);
+	mDLOutput->SetScheduledFrameCompletionCallback(NULL);
 	mDLOutput->DisableVideoOutput();
+	mAudioPrerolling = false;
+	if (mAudioEnabled)
+		mDLOutput->DisableAudioOutput();
 
 	return true;
 }
@@ -1411,6 +1598,9 @@ bool OpenGLComposite::compileSingleLayerProgram(const RuntimeRenderState& state,
 	const GLint videoInputLocation = glGetUniformLocation(newProgram.get(), "gVideoInput");
 	if (videoInputLocation >= 0)
 		glUniform1i(videoInputLocation, static_cast<GLint>(kDecodedVideoTextureUnit));
+	const GLint audioDataLocation = glGetUniformLocation(newProgram.get(), "gAudioData");
+	if (audioDataLocation >= 0)
+		glUniform1i(audioDataLocation, static_cast<GLint>(kAudioDataTextureUnit));
 	for (unsigned index = 0; index < historyCap; ++index)
 	{
 		const std::string sourceSamplerName = "gSourceHistory" + std::to_string(index);
@@ -1973,6 +2163,8 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
 	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 	glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit);
 	glBindTexture(GL_TEXTURE_2D, sourceTexture);
+	glActiveTexture(GL_TEXTURE0 + kAudioDataTextureUnit);
+	glBindTexture(GL_TEXTURE_2D, mAudioDataTexture);
 	bindHistorySamplers(state, sourceTexture);
 	bindLayerTextureAssets(layerProgram);
 	glBindVertexArray(mFullscreenVAO);
@@ -1995,6 +2187,8 @@ void OpenGLComposite::renderShaderProgram(GLuint sourceTexture, GLuint destinati
 		glActiveTexture(GL_TEXTURE0 + shaderTextureBase + static_cast<GLuint>(index));
 		glBindTexture(GL_TEXTURE_2D, 0);
 	}
+	glActiveTexture(GL_TEXTURE0 + kAudioDataTextureUnit);
+	glBindTexture(GL_TEXTURE_2D, 0);
 	glActiveTexture(GL_TEXTURE0 + kDecodedVideoTextureUnit);
 	glBindTexture(GL_TEXTURE_2D, 0);
 	glActiveTexture(GL_TEXTURE0);
@@ -2066,6 +2260,55 @@ void OpenGLComposite::broadcastRuntimeState()
 		mControlServer->BroadcastState();
 }
 
+unsigned OpenGLComposite::delayedAudioSampleFrames() const
+{
+	return static_cast<unsigned>(AudioSampleTimeForVideoFrame(kVideoPrerollFrameCount, mFrameDuration, mFrameTimescale));
+}
+
+void OpenGLComposite::initializeAudioDelay()
+{
+	std::lock_guard<std::mutex> audioLock(mAudioStateMutex);
+	mAudioAnalyzer.Reset();
+	mAudioAnalysis = AudioAnalysisSnapshot();
+	mAudioDelayBuffer.Reset(delayedAudioSampleFrames());
+	mNextAudioSampleFrame = 0;
+	updateAudioStatus();
+}
+
+AudioFrameBlock OpenGLComposite::popAudioForVideoFrame(uint64_t videoFrameIndex)
+{
+	const unsigned sampleFrames = AudioSamplesForVideoFrame(videoFrameIndex, mFrameDuration, mFrameTimescale);
+	bool underrun = false;
+	AudioFrameBlock block = mAudioDelayBuffer.Pop(sampleFrames, underrun);
+	mAudioAnalysis = mAudioAnalyzer.Analyze(block);
+	return block;
+}
+
+void OpenGLComposite::updateAudioDataTexture(const AudioAnalysisSnapshot& analysis)
+{
+	if (mAudioDataTexture == 0)
+		return;
+
+	glActiveTexture(GL_TEXTURE0 + kAudioDataTextureUnit);
+	glBindTexture(GL_TEXTURE_2D, mAudioDataTexture);
+	glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, kAudioTextureWidth, kAudioTextureHeight, GL_RGBA, GL_FLOAT, analysis.texture.data());
+	glBindTexture(GL_TEXTURE_2D, 0);
+	glActiveTexture(GL_TEXTURE0);
+}
+
+void OpenGLComposite::updateAudioStatus()
+{
+	if (!mRuntimeHost)
+		return;
+
+	AudioStatusSnapshot status;
+	status.enabled = mAudioEnabled;
+	status.bufferedSampleFrames = mAudioDelayBuffer.BufferedSampleFrames();
+	status.underrunCount = mAudioDelayBuffer.UnderrunCount();
+	status.analysis = mAudioAnalysis;
+	mRuntimeHost->SetAudioStatus(status);
+}
+
 bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength)
 {
 	std::vector<unsigned char> buffer;
@@ -2085,6 +2328,15 @@ bool OpenGLComposite::updateGlobalParamsBuffer(const RuntimeRenderState& state,
 		: 0u;
 	AppendStd140Int(buffer, static_cast<int>(effectiveSourceHistoryLength));
 	AppendStd140Int(buffer, static_cast<int>(effectiveTemporalHistoryLength));
+	AppendStd140Vec2(buffer, state.audioAnalysis.rms[0], state.audioAnalysis.rms[1]);
+	AppendStd140Vec2(buffer, state.audioAnalysis.peak[0], state.audioAnalysis.peak[1]);
+	AppendStd140Float(buffer, state.audioAnalysis.monoRms);
+	AppendStd140Float(buffer, state.audioAnalysis.monoPeak);
+	AppendStd140Vec4(buffer,
+		state.audioAnalysis.bands[0],
+		state.audioAnalysis.bands[1],
+		state.audioAnalysis.bands[2],
+		state.audioAnalysis.bands[3]);
 
 	for (const ShaderParameterDefinition& definition : state.parameterDefinitions)
 	{
@@ -2623,11 +2875,14 @@ ULONG	CaptureDelegate::Release()
 	return newCount;
 }
 
-HRESULT	CaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* inputFrame, IDeckLinkAudioInputPacket* /*audioPacket*/)
+HRESULT	CaptureDelegate::VideoInputFrameArrived(IDeckLinkVideoInputFrame* inputFrame, IDeckLinkAudioInputPacket* audioPacket)
 {
+	if (audioPacket)
+		m_pOwner->AudioPacketArrived(audioPacket);
+
 	if (! inputFrame)
 	{
-		// It's possible to receive a NULL inputFrame, but a valid audioPacket. Ignore audio-only frame.
+		// It's possible to receive a NULL inputFrame, but a valid audioPacket.
 		return S_OK;
 	}
 
@@ -2653,6 +2908,23 @@ PlayoutDelegate::PlayoutDelegate(OpenGLComposite* pOwner) :
 
 HRESULT	PlayoutDelegate::QueryInterface(REFIID iid, LPVOID *ppv)
 {
+	if (ppv == nullptr)
+		return E_POINTER;
+
+	if (iid == IID_IUnknown || iid == IID_IDeckLinkVideoOutputCallback)
+	{
+		*ppv = static_cast<IDeckLinkVideoOutputCallback*>(this);
+		AddRef();
+		return S_OK;
+	}
+
+	if (iid == IID_IDeckLinkAudioOutputCallback)
+	{
+		*ppv = static_cast<IDeckLinkAudioOutputCallback*>(this);
+		AddRef();
+		return S_OK;
+	}
+
 	*ppv = NULL;
 	return E_NOINTERFACE;
 }
@@ -2694,3 +2966,8 @@ HRESULT	PlayoutDelegate::ScheduledPlaybackHasStopped ()
 {
 	return S_OK;
 }
+
+HRESULT	PlayoutDelegate::RenderAudioSamples (BOOL preroll)
+{
+	return m_pOwner->RenderAudioSamples(preroll);
+}

apps/LoopThroughWithOpenGLCompositing/OpenGLComposite.h

@@ -52,13 +52,16 @@
 #include <comutil.h>
 #include "DeckLinkAPI_h.h"
 
+#include "AudioSupport.h"
 #include "VideoFrameTransfer.h"
 #include "RuntimeHost.h"
 
 #include <atomic>
+#include <cstdint>
 #include <functional>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <vector>
 #include <deque>
 
@@ -96,6 +99,8 @@ public:
 	void paintGL();
 
 	void VideoFrameArrived(IDeckLinkVideoInputFrame* inputFrame, bool hasNoInputSource);
+	void AudioPacketArrived(IDeckLinkAudioInputPacket* audioPacket);
+	HRESULT RenderAudioSamples(BOOL preroll);
 	void PlayoutFrameCompleted(IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult result);
 
 private:
@@ -112,12 +117,14 @@ private:
 	// DeckLink
 	IDeckLinkInput*							mDLInput;
 	IDeckLinkOutput*						mDLOutput;
+	IDeckLinkConfiguration*					mDLInputConfiguration;
 	IDeckLinkKeyer*							mDLKeyer;
 	std::deque<IDeckLinkMutableVideoFrame*>	mDLOutputVideoFrameQueue;
 	PinnedMemoryAllocator*					mPlayoutAllocator;
 	BMDTimeValue							mFrameDuration;
 	BMDTimeScale							mFrameTimescale;
 	unsigned								mTotalPlayoutFrames;
+	uint64_t								mNextAudioSampleFrame;
 	unsigned								mInputFrameWidth;
 	unsigned								mInputFrameHeight;
 	unsigned								mOutputFrameWidth;
@@ -139,6 +146,7 @@ private:
 	GLuint									mLayerTempTexture;
 	GLuint									mFBOTexture;
 	GLuint									mOutputTexture;
+	GLuint									mAudioDataTexture;
 	GLuint									mUnpinnedTextureBuffer;
 	GLuint									mDecodeFrameBuf;
 	GLuint									mLayerTempFrameBuf;
@@ -157,6 +165,12 @@ private:
 	std::unique_ptr<RuntimeHost>				mRuntimeHost;
 	std::unique_ptr<ControlServer>			mControlServer;
 	std::unique_ptr<OscServer>				mOscServer;
+	bool									mAudioEnabled;
+	bool									mAudioPrerolling;
+	std::mutex								mAudioStateMutex;
+	AudioDelayBuffer						mAudioDelayBuffer;
+	AudioAnalyzer							mAudioAnalyzer;
+	AudioAnalysisSnapshot					mAudioAnalysis;
 
 	struct LayerProgram
 	{
@@ -209,6 +223,11 @@ private:
 	void renderEffect();
 	bool PollRuntimeChanges();
 	void broadcastRuntimeState();
+	void initializeAudioDelay();
+	unsigned delayedAudioSampleFrames() const;
+	AudioFrameBlock popAudioForVideoFrame(uint64_t videoFrameIndex);
+	void updateAudioDataTexture(const AudioAnalysisSnapshot& analysis);
+	void updateAudioStatus();
 	bool updateGlobalParamsBuffer(const RuntimeRenderState& state, unsigned availableSourceHistoryLength, unsigned availableTemporalHistoryLength);
 	bool validateTemporalTextureUnitBudget(const std::vector<RuntimeRenderState>& layerStates, std::string& error) const;
 	bool ensureTemporalHistoryResources(const std::vector<RuntimeRenderState>& layerStates, std::string& error);
@@ -341,7 +360,7 @@ public:
 // Render Delegate Class
 ////////////////////////////////////////////
 
-class PlayoutDelegate : public IDeckLinkVideoOutputCallback
+class PlayoutDelegate : public IDeckLinkVideoOutputCallback, public IDeckLinkAudioOutputCallback
 {
 	OpenGLComposite*	m_pOwner;
 	LONG				mRefCount;
@@ -356,6 +375,7 @@ public:
 
 	virtual HRESULT	STDMETHODCALLTYPE	ScheduledFrameCompleted (IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult result);
 	virtual HRESULT	STDMETHODCALLTYPE	ScheduledPlaybackHasStopped ();
+	virtual HRESULT	STDMETHODCALLTYPE	RenderAudioSamples (BOOL preroll);
 };
 
 #endif		// __OPENGL_COMPOSITE_H__

apps/LoopThroughWithOpenGLCompositing/RuntimeHost.cpp

@@ -1055,6 +1055,12 @@ void RuntimeHost::SetPerformanceStats(double frameBudgetMilliseconds, double ren
 		mSmoothedRenderMilliseconds = mSmoothedRenderMilliseconds * 0.9 + mRenderMilliseconds * 0.1;
 }
 
+void RuntimeHost::SetAudioStatus(const AudioStatusSnapshot& status)
+{
+	std::lock_guard<std::mutex> lock(mMutex);
+	mAudioStatus = status;
+}
+
 void RuntimeHost::AdvanceFrame()
 {
 	std::lock_guard<std::mutex> lock(mMutex);
@@ -1121,6 +1127,7 @@ std::vector<RuntimeRenderState> RuntimeHost::GetLayerRenderStates(unsigned outpu
 		state.inputHeight = mSignalHeight;
 		state.outputWidth = outputWidth;
 		state.outputHeight = outputHeight;
+		state.audioAnalysis = mAudioStatus.analysis;
 		state.parameterDefinitions = shaderIt->second.parameters;
 		state.textureAssets = shaderIt->second.textureAssets;
 		state.isTemporal = shaderIt->second.temporal.enabled;
@@ -1182,6 +1189,21 @@ bool RuntimeHost::LoadConfig(std::string& error)
 	}
 	if (const JsonValue* enableExternalKeyingValue = configJson.find("enableExternalKeying"))
 		mConfig.enableExternalKeying = enableExternalKeyingValue->asBoolean(mConfig.enableExternalKeying);
+	if (const JsonValue* audioEnabledValue = configJson.find("audioEnabled"))
+		mConfig.audioEnabled = audioEnabledValue->asBoolean(mConfig.audioEnabled);
+	if (const JsonValue* audioChannelCountValue = configJson.find("audioChannelCount"))
+		mConfig.audioChannelCount = static_cast<unsigned>(audioChannelCountValue->asNumber(static_cast<double>(mConfig.audioChannelCount)));
+	if (const JsonValue* audioSampleRateValue = configJson.find("audioSampleRate"))
+		mConfig.audioSampleRate = static_cast<unsigned>(audioSampleRateValue->asNumber(static_cast<double>(mConfig.audioSampleRate)));
+	if (const JsonValue* audioDelayModeValue = configJson.find("audioDelayMode"))
+	{
+		if (audioDelayModeValue->isString() && !audioDelayModeValue->asString().empty())
+			mConfig.audioDelayMode = audioDelayModeValue->asString();
+	}
+	if (mConfig.audioChannelCount != kAudioChannelCount)
+		mConfig.audioChannelCount = kAudioChannelCount;
+	if (mConfig.audioSampleRate != kAudioSampleRate)
+		mConfig.audioSampleRate = kAudioSampleRate;
 	if (const JsonValue* videoFormatValue = configJson.find("videoFormat"))
 	{
 		if (videoFormatValue->isString() && !videoFormatValue->asString().empty())
@@ -1519,6 +1541,10 @@ JsonValue RuntimeHost::BuildStateValue() const
 	app.set("autoReload", JsonValue(mAutoReloadEnabled));
 	app.set("maxTemporalHistoryFrames", JsonValue(static_cast<double>(mConfig.maxTemporalHistoryFrames)));
 	app.set("enableExternalKeying", JsonValue(mConfig.enableExternalKeying));
+	app.set("audioEnabled", JsonValue(mConfig.audioEnabled));
+	app.set("audioChannelCount", JsonValue(static_cast<double>(mConfig.audioChannelCount)));
+	app.set("audioSampleRate", JsonValue(static_cast<double>(mConfig.audioSampleRate)));
+	app.set("audioDelayMode", JsonValue(mConfig.audioDelayMode));
 	app.set("inputVideoFormat", JsonValue(mConfig.inputVideoFormat));
 	app.set("inputFrameRate", JsonValue(mConfig.inputFrameRate));
 	app.set("outputVideoFormat", JsonValue(mConfig.outputVideoFormat));
@@ -1538,6 +1564,26 @@ JsonValue RuntimeHost::BuildStateValue() const
 	video.set("modeName", JsonValue(mSignalModeName));
 	root.set("video", video);
 
+	JsonValue audio = JsonValue::MakeObject();
+	audio.set("enabled", JsonValue(mAudioStatus.enabled));
+	audio.set("bufferedSampleFrames", JsonValue(static_cast<double>(mAudioStatus.bufferedSampleFrames)));
+	audio.set("underrunCount", JsonValue(static_cast<double>(mAudioStatus.underrunCount)));
+	JsonValue rms = JsonValue::MakeArray();
+	rms.pushBack(JsonValue(static_cast<double>(mAudioStatus.analysis.rms[0])));
+	rms.pushBack(JsonValue(static_cast<double>(mAudioStatus.analysis.rms[1])));
+	audio.set("rms", rms);
+	JsonValue peak = JsonValue::MakeArray();
+	peak.pushBack(JsonValue(static_cast<double>(mAudioStatus.analysis.peak[0])));
+	peak.pushBack(JsonValue(static_cast<double>(mAudioStatus.analysis.peak[1])));
+	audio.set("peak", peak);
+	audio.set("monoRms", JsonValue(static_cast<double>(mAudioStatus.analysis.monoRms)));
+	audio.set("monoPeak", JsonValue(static_cast<double>(mAudioStatus.analysis.monoPeak)));
+	JsonValue bands = JsonValue::MakeArray();
+	for (float band : mAudioStatus.analysis.bands)
+		bands.pushBack(JsonValue(static_cast<double>(band)));
+	audio.set("bands", bands);
+	root.set("audio", audio);
+
 	JsonValue deckLink = JsonValue::MakeObject();
 	deckLink.set("modelName", JsonValue(mDeckLinkOutputStatus.modelName));
 	deckLink.set("supportsInternalKeying", JsonValue(mDeckLinkOutputStatus.supportsInternalKeying));

apps/LoopThroughWithOpenGLCompositing/RuntimeHost.h

@@ -38,6 +38,7 @@ public:
 	void SetDeckLinkOutputStatus(const std::string& modelName, bool supportsInternalKeying, bool supportsExternalKeying,
 		bool keyerInterfaceAvailable, bool externalKeyingRequested, bool externalKeyingActive, const std::string& statusMessage);
 	void SetPerformanceStats(double frameBudgetMilliseconds, double renderMilliseconds);
+	void SetAudioStatus(const AudioStatusSnapshot& status);
 	void AdvanceFrame();
 
 	bool BuildLayerFragmentShaderSource(const std::string& layerId, std::string& fragmentShaderSource, std::string& error);
@@ -52,6 +53,9 @@ public:
 	unsigned short GetOscPort() const { return mConfig.oscPort; }
 	unsigned GetMaxTemporalHistoryFrames() const { return mConfig.maxTemporalHistoryFrames; }
 	bool ExternalKeyingEnabled() const { return mConfig.enableExternalKeying; }
+	bool AudioEnabled() const { return mConfig.audioEnabled; }
+	unsigned AudioChannelCount() const { return mConfig.audioChannelCount; }
+	unsigned AudioSampleRate() const { return mConfig.audioSampleRate; }
 	const std::string& GetInputVideoFormat() const { return mConfig.inputVideoFormat; }
 	const std::string& GetInputFrameRate() const { return mConfig.inputFrameRate; }
 	const std::string& GetOutputVideoFormat() const { return mConfig.outputVideoFormat; }
@@ -68,6 +72,10 @@ private:
 		bool autoReload = true;
 		unsigned maxTemporalHistoryFrames = 4;
 		bool enableExternalKeying = false;
+		bool audioEnabled = true;
+		unsigned audioChannelCount = kAudioChannelCount;
+		unsigned audioSampleRate = kAudioSampleRate;
+		std::string audioDelayMode = "matchVideoPreroll";
 		std::string inputVideoFormat = "1080p";
 		std::string inputFrameRate = "59.94";
 		std::string outputVideoFormat = "1080p";
@@ -148,6 +156,7 @@ private:
 	double mRenderMilliseconds;
 	double mSmoothedRenderMilliseconds;
 	DeckLinkOutputStatus mDeckLinkOutputStatus;
+	AudioStatusSnapshot mAudioStatus;
 	unsigned short mServerPort;
 	bool mAutoReloadEnabled;
 	std::chrono::steady_clock::time_point mStartTime;

apps/LoopThroughWithOpenGLCompositing/ShaderTypes.h

@@ -5,6 +5,8 @@
 #include <string>
 #include <vector>
 
+#include "AudioSupport.h"
+
 enum class ShaderParameterType
 {
 	Float,
@@ -95,6 +97,7 @@ struct RuntimeRenderState
 	unsigned inputHeight = 0;
 	unsigned outputWidth = 0;
 	unsigned outputHeight = 0;
+	AudioAnalysisSnapshot audioAnalysis;
 	bool isTemporal = false;
 	TemporalHistorySource temporalHistorySource = TemporalHistorySource::None;
 	unsigned requestedTemporalHistoryLength = 0;