From de9b09251df719475f2bafeca1b6d7307bdab752 Mon Sep 17 00:00:00 2001 From: Christopher Snowhill Date: Fri, 15 Jul 2022 03:34:10 -0700 Subject: [PATCH] [Audio Output] Greatly improve sample rate changes Sample rate changes will now occur on exact sample boundaries, like they are supposed to. Also, FreeSurround accounts for its output latency. Signed-off-by: Christopher Snowhill --- Audio/Chain/OutputNode.m | 8 +++ Audio/Output/FSurroundFilter.h | 1 + Audio/Output/FSurroundFilter.mm | 4 ++ Audio/Output/OutputAVFoundation.h | 4 +- Audio/Output/OutputAVFoundation.m | 102 ++++++++++++++++++++++-------- 5 files changed, 93 insertions(+), 26 deletions(-) diff --git a/Audio/Chain/OutputNode.m b/Audio/Chain/OutputNode.m index 6be97d72e..6a9e07306 100644 --- a/Audio/Chain/OutputNode.m +++ b/Audio/Chain/OutputNode.m @@ -103,6 +103,14 @@ } } +- (BOOL)peekFormat:(nonnull AudioStreamBasicDescription *)format channelConfig:(nonnull uint32_t *)config { + @autoreleasepool { + [self setPreviousNode:[[controller bufferChain] finalNode]]; + + return [super peekFormat:format channelConfig:config]; + } +} + - (double)amountPlayed { return amountPlayed; } diff --git a/Audio/Output/FSurroundFilter.h b/Audio/Output/FSurroundFilter.h index 9898490a0..19812ba82 100644 --- a/Audio/Output/FSurroundFilter.h +++ b/Audio/Output/FSurroundFilter.h @@ -26,6 +26,7 @@ - (uint32_t)channelCount; - (uint32_t)channelConfig; +- (double)srate; - (void)process:(const float *)samplesIn output:(float *)samplesOut count:(uint32_t)count; diff --git a/Audio/Output/FSurroundFilter.mm b/Audio/Output/FSurroundFilter.mm index 24e02de82..f1f3f1393 100644 --- a/Audio/Output/FSurroundFilter.mm +++ b/Audio/Output/FSurroundFilter.mm @@ -121,6 +121,10 @@ struct freesurround_params { return channelConfig; } +- (double)srate { + return srate; +} + - (void)process:(const float *)samplesIn output:(float *)samplesOut count:(uint32_t)count { freesurround_params *_params = (freesurround_params *)params; freesurround_decoder *_decoder = (freesurround_decoder *)decoder; diff --git a/Audio/Output/OutputAVFoundation.h b/Audio/Output/OutputAVFoundation.h index 4a0a1fae7..e490640d5 100644 --- a/Audio/Output/OutputAVFoundation.h +++ b/Audio/Output/OutputAVFoundation.h @@ -111,6 +111,8 @@ using std::atomic_long; HeadphoneFilter *hrtf; BOOL enableFSurround; + BOOL FSurroundDelayRemoved; + int inputBufferLastTime; FSurroundFilter *fsurround; BOOL resetStreamFormat; @@ -119,7 +121,7 @@ using std::atomic_long; float tempBuffer[512 * 32]; float r8bTempBuffer[4096 * 32]; float inputBuffer[4096 * 32]; // 4096 samples times maximum supported channel count - float fsurroundBuffer[4096 * 6]; + float fsurroundBuffer[8192 * 6]; float hrtfBuffer[4096 * 2]; float eqBuffer[4096 * 32]; diff --git a/Audio/Output/OutputAVFoundation.m b/Audio/Output/OutputAVFoundation.m index 492884445..9949fc37f 100644 --- a/Audio/Output/OutputAVFoundation.m +++ b/Audio/Output/OutputAVFoundation.m @@ -69,17 +69,13 @@ static OSStatus eqRenderCallback(void *inRefCon, AudioUnitRenderActionFlags *ioA return 0; } - AudioChunk *chunk = [outputController readChunk:amountToRead]; - - int frameCount = (int)[chunk frameCount]; - AudioStreamBasicDescription format = [chunk format]; - uint32_t config = [chunk channelConfig]; - double chunkDuration = 0; - - if(frameCount) { + AudioStreamBasicDescription format; + uint32_t config; + if([outputController peekFormat:&format channelConfig:&config]) { // XXX ERROR with AirPods - Can't go higher than CD*8 surround - 192k stereo // Emits to console: [AUScotty] Initialize: invalid FFT size 16384 // DSD256 stereo emits: [AUScotty] Initialize: invalid FFT size 65536 + BOOL formatClipped = NO; BOOL isSurround = format.mChannelsPerFrame > 2; const double maxSampleRate = isSurround ? 352800.0 : 192000.0; @@ -117,10 +113,23 @@ static OSStatus eqRenderCallback(void *inRefCon, AudioUnitRenderActionFlags *ioA if(!r8bold) { realStreamFormat = format; realStreamChannelConfig = config; - [self updateStreamFormat]; + streamFormatChanged = YES; } } + } + if(streamFormatChanged) { + return 0; + } + + AudioChunk *chunk = [outputController readChunk:amountToRead]; + + int frameCount = (int)[chunk frameCount]; + format = [chunk format]; + config = [chunk channelConfig]; + double chunkDuration = 0; + + if(frameCount) { chunkDuration = [chunk duration]; NSData *samples = [chunk removeSamples:frameCount]; @@ -301,10 +310,12 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons eqPreamp = pow(10.0, preamp / 20.0); } else if([keyPath isEqualToString:@"values.enableHrtf"]) { enableHrtf = [[[NSUserDefaultsController sharedUserDefaultsController] defaults] boolForKey:@"enableHrtf"]; - resetStreamFormat = YES; + if(streamFormatStarted) + resetStreamFormat = YES; } else if([keyPath isEqualToString:@"values.enableFSurround"]) { enableFSurround = [[[NSUserDefaultsController sharedUserDefaultsController] defaults] boolForKey:@"enableFSurround"]; - resetStreamFormat = YES; + if(streamFormatStarted) + resetStreamFormat = YES; } } @@ -600,18 +611,22 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons - (void)updateStreamFormat { /* Set the channel layout for the audio queue */ - streamFormatChanged = YES; resetStreamFormat = NO; uint32_t channels = realStreamFormat.mChannelsPerFrame; uint32_t channelConfig = realStreamChannelConfig; if(enableFSurround && channels == 2 && channelConfig == AudioConfigStereo) { + [currentPtsLock lock]; fsurround = [[FSurroundFilter alloc] initWithSampleRate:realStreamFormat.mSampleRate]; + [currentPtsLock unlock]; channels = [fsurround channelCount]; channelConfig = [fsurround channelConfig]; + FSurroundDelayRemoved = NO; } else { + [currentPtsLock lock]; fsurround = nil; + [currentPtsLock unlock]; } /* Apple's audio processor really only supports common 1-8 channel formats */ @@ -742,13 +757,15 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons status = CMBlockBufferCreateEmpty(kCFAllocatorDefault, 0, 0, &blockListBuffer); if(status != noErr || !blockListBuffer) return 0; - if(resetStreamFormat) { + if(resetStreamFormat || streamFormatChanged) { + streamFormatChanged = NO; [self updateStreamFormat]; } - int inputRendered = 0; - int bytesRendered = 0; - do { + int inputRendered = inputBufferLastTime; + int bytesRendered = inputRendered * newFormat.mBytesPerPacket; + + while(inputRendered < 4096) { int maxToRender = MIN(4096 - inputRendered, 512); int rendered = [self renderInput:maxToRender toBuffer:&tempBuffer[0]]; if(rendered > 0) { @@ -758,12 +775,17 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons bytesRendered += rendered * newFormat.mBytesPerPacket; if(streamFormatChanged) { streamFormatChanged = NO; - if(rendered < maxToRender) { + if(inputRendered) { + resetStreamFormat = YES; break; + } else { + [self updateStreamFormat]; } } if([self processEndOfStream]) break; - } while(inputRendered < 4096); + } + + inputBufferLastTime = inputRendered; int samplesRenderedTotal = 0; @@ -793,16 +815,37 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons r8bDone = NO; realStreamFormat = newFormat; realStreamChannelConfig = newChannelConfig; - [self updateStreamFormat]; + streamFormatChanged = YES; } } - if(samplesRendered) { + if(samplesRendered || fsurround) { if(fsurround) { - [fsurround process:samplePtr output:&fsurroundBuffer[0] count:samplesRendered]; + int countToProcess = samplesRendered; + if(countToProcess < 4096) { + bzero(samplePtr + countToProcess * 2, (4096 - countToProcess) * 2 * sizeof(float)); + countToProcess = 4096; + } + [fsurround process:samplePtr output:&fsurroundBuffer[0] count:countToProcess]; samplePtr = &fsurroundBuffer[0]; + if(resetStreamFormat || samplesRendered < 4096) { + bzero(&fsurroundBuffer[4096 * 6], 4096 * 2 * sizeof(float)); + [fsurround process:&fsurroundBuffer[4096 * 6] output:&fsurroundBuffer[4096 * 6] count:4096]; + samplesRendered += 2048; + } + if(!FSurroundDelayRemoved) { + FSurroundDelayRemoved = YES; + if(samplesRendered > 2048) { + samplePtr += 2048 * 6; + samplesRendered -= 2048; + } + } } - + + if(!samplesRendered) { + break; + } + if(hrtf) { [hrtf process:samplePtr sampleCount:samplesRendered toBuffer:&hrtfBuffer[0]]; samplePtr = &hrtfBuffer[0]; @@ -871,11 +914,13 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons } if(i == 0) { - if(!samplesRendered) { - *blockBufferOut = blockListBuffer; - return samplesRenderedTotal + samplesRendered; + samplesRenderedTotal += samplesRendered; + if(samplesRendered) { + break; } + ++i; } else { + inputBufferLastTime = 0; samplesRenderedTotal += samplesRendered; ++i; } @@ -898,6 +943,9 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons audioFormatDescription = NULL; resetStreamFormat = NO; + streamFormatChanged = NO; + + inputBufferLastTime = 0; running = NO; stopping = NO; @@ -1031,6 +1079,7 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons void **r8bstate = &self->r8bstate; void **r8bold = &self->r8bold; void **r8bvis = &self->r8bvis; + FSurroundFilter *const *fsurroundtest = &self->fsurround; currentPtsObserver = [renderSynchronizer addPeriodicTimeObserverForInterval:interval queue:NULL usingBlock:^(CMTime time) { @@ -1057,6 +1106,9 @@ current_device_listener(AudioObjectID inObjectID, UInt32 inNumberAddresses, cons if(*r8bvis) { latencyVis = r8bstate_latency(*r8bvis); } + if(*fsurroundtest) { + latencyVis += 2048.0 / [(*fsurroundtest) srate]; + } [lock unlock]; if(latencySeconds < 0) latencySeconds = 0;