Android Audio Reactivity

Assets/Editor/BuildTiltBrush.cs

@@ -1512,7 +1512,7 @@ public static void DoBuild(TiltBuildOptions tiltOptions)
             ? StereoRenderingPath.SinglePass : StereoRenderingPath.MultiPass))
         using (var unused3 = new TempDefineSymbols(
             target,
-            tiltOptions.Il2Cpp ? "DISABLE_AUDIO_CAPTURE" : null,
+            tiltOptions.Il2Cpp ? "DISABLE_SYSTEM_AUDIO_CAPTURE" : null,
             tiltOptions.AutoProfile ? "AUTOPROFILE_ENABLED" : null))
         using (var unused4 = new TempHookUpSingletons())
         using (var unused5 = new TempSetScriptingBackend(target, tiltOptions.Il2Cpp))

Assets/Editor/Tests/TestVisualizerManagedAnalysis.cs

@@ -0,0 +1,141 @@
+// Copyright 2020 The Tilt Brush Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using NUnit.Framework;
+
+namespace TiltBrush
+{
+    internal class TestVisualizerManagedAnalysis
+    {
+        private const int kFftSize = 512;
+        private const int kSampleRate = 48000;
+
+        [Test]
+        public void ManagedFftDetectsSineNearExpectedBin()
+        {
+            const int expectedBin = 8;
+            float[] samples = GenerateSine(expectedBin * kSampleRate / kFftSize);
+            float[] fft = new float[kFftSize];
+            var analyzer = new VisualizerManagedFft(1, kFftSize);
+
+            analyzer.Add(samples, samples.Length);
+            analyzer.GetFftData(fft);
+
+            int peakBin = FindPeakBin(fft, 1, kFftSize / 2);
+            Assert.That(peakBin, Is.InRange(expectedBin - 1, expectedBin + 1));
+            Assert.That(fft[peakBin], Is.GreaterThan(fft[expectedBin + 8] * 5.0f));
+        }
+
+        [Test]
+        public void ManagedFftKeepsSilenceAtZero()
+        {
+            float[] samples = new float[kFftSize];
+            float[] fft = new float[kFftSize];
+            var analyzer = new VisualizerManagedFft(1, kFftSize);
+
+            analyzer.Add(samples, samples.Length);
+            analyzer.GetFftData(fft);
+
+            for (int i = 0; i < fft.Length; ++i)
+            {
+                Assert.That(fft[i], Is.EqualTo(0.0f).Within(1e-6f));
+            }
+        }
+
+        [Test]
+        public void ManagedFftUsesFirstChannelFromInterleavedSamples()
+        {
+            const int expectedBin = 8;
+            float[] mono = GenerateSine(expectedBin * kSampleRate / kFftSize);
+            float[] interleaved = new float[kFftSize * 2];
+            for (int i = 0; i < mono.Length; ++i)
+            {
+                interleaved[i * 2] = mono[i];
+                interleaved[i * 2 + 1] = 0.0f;
+            }
+            float[] fft = new float[kFftSize];
+            var analyzer = new VisualizerManagedFft(2, kFftSize);
+
+            analyzer.Add(interleaved, interleaved.Length);
+            analyzer.GetFftData(fft);
+
+            int peakBin = FindPeakBin(fft, 1, kFftSize / 2);
+            Assert.That(peakBin, Is.InRange(expectedBin - 1, expectedBin + 1));
+        }
+
+        [Test]
+        public void ManagedLowPassAttenuatesHighFrequencies()
+        {
+            float[] low = GenerateSine(120);
+            float[] high = GenerateSine(4000);
+            var lowPass = new VisualizerManagedFilter(VisualizerManagedFilter.FilterType.Low, kSampleRate, 500);
+
+            lowPass.Process(low);
+            lowPass = new VisualizerManagedFilter(VisualizerManagedFilter.FilterType.Low, kSampleRate, 500);
+            lowPass.Process(high);
+
+            Assert.That(Rms(low), Is.GreaterThan(Rms(high) * 2.0f));
+        }
+
+        [Test]
+        public void ManagedHighPassAttenuatesLowFrequencies()
+        {
+            float[] low = GenerateSine(120);
+            float[] high = GenerateSine(4000);
+            var highPass = new VisualizerManagedFilter(VisualizerManagedFilter.FilterType.High, kSampleRate, 1000);
+
+            highPass.Process(low);
+            highPass = new VisualizerManagedFilter(VisualizerManagedFilter.FilterType.High, kSampleRate, 1000);
+            highPass.Process(high);
+
+            Assert.That(Rms(high), Is.GreaterThan(Rms(low) * 2.0f));
+        }
+
+        private static float[] GenerateSine(float frequency)
+        {
+            float[] samples = new float[kFftSize];
+            for (int i = 0; i < samples.Length; ++i)
+            {
+                samples[i] = (float)Math.Sin(2.0 * Math.PI * frequency * i / kSampleRate);
+            }
+            return samples;
+        }
+
+        private static int FindPeakBin(float[] fft, int start, int end)
+        {
+            int peakIndex = start;
+            float peak = fft[start];
+            for (int i = start + 1; i < end; ++i)
+            {
+                if (fft[i] > peak)
+                {
+                    peak = fft[i];
+                    peakIndex = i;
+                }
+            }
+            return peakIndex;
+        }
+
+        private static float Rms(float[] samples)
+        {
+            double sumSquares = 0.0;
+            for (int i = 0; i < samples.Length; ++i)
+            {
+                sumSquares += samples[i] * samples[i];
+            }
+            return (float)Math.Sqrt(sumSquares / samples.Length);
+        }
+    }
+}

Assets/Scripts/AppAudioMonitor.cs

@@ -19,18 +19,101 @@ namespace TiltBrush
 
     public class AppAudioMonitor : MonoBehaviour
     {
+        private const string kAndroidAppAudioLogPrefix = "AR_ANDROID_APP_AUDIO_20260603";
+        private const float kAndroidLogInterval = 2.0f;
+
+        [SerializeField] private bool m_DebugPlayTestTone;
+        [SerializeField] private float m_DebugTestToneFrequency = 440.0f;
+        [SerializeField] private float m_DebugTestToneVolume = 0.05f;
+
         private float[] m_WaveformFloats;
+        private float m_NextAndroidLogTime;
+        private int m_NonZeroLogCount;
 
         void Start()
         {
             m_WaveformFloats = new float[VisualizerManager.m_Instance.FFTSize];
+#if UNITY_EDITOR || DEVELOPMENT_BUILD
+            if (m_DebugPlayTestTone)
+            {
+                StartDebugTestTone();
+            }
+#endif
+        }
+
+#if UNITY_ANDROID
+        void OnEnable()
+        {
+            Debug.Log($"{kAndroidAppAudioLogPrefix} AppAudioMonitor enabled sampleRate={AudioSettings.outputSampleRate}");
+        }
+
+        void OnDisable()
+        {
+            Debug.Log($"{kAndroidAppAudioLogPrefix} AppAudioMonitor disabled");
         }
+#endif
 
         void Update()
         {
             AudioListener.GetOutputData(m_WaveformFloats, 0);
+#if UNITY_ANDROID
+            LogAndroidAudioSamples();
+#endif
             VisualizerManager.m_Instance.ProcessAudio(m_WaveformFloats, AudioSettings.outputSampleRate);
         }
+
+#if UNITY_ANDROID
+        private void LogAndroidAudioSamples()
+        {
+            float peak = 0.0f;
+            float sumSquares = 0.0f;
+            for (int i = 0; i < m_WaveformFloats.Length; ++i)
+            {
+                float sample = m_WaveformFloats[i];
+                peak = Mathf.Max(peak, Mathf.Abs(sample));
+                sumSquares += sample * sample;
+            }
+
+            float rms = Mathf.Sqrt(sumSquares / m_WaveformFloats.Length);
+            bool shouldLog = m_NonZeroLogCount < 3 && peak > 0.0001f;
+            if (Time.unscaledTime >= m_NextAndroidLogTime || shouldLog)
+            {
+                bool audioReactiveEnabled = Shader.IsKeywordEnabled("AUDIO_REACTIVE");
+                Debug.Log($"{kAndroidAppAudioLogPrefix} AppAudioMonitor samples peak={peak:F5} rms={rms:F5} sampleRate={AudioSettings.outputSampleRate} AUDIO_REACTIVE={audioReactiveEnabled}");
+                m_NextAndroidLogTime = Time.unscaledTime + kAndroidLogInterval;
+                if (peak > 0.0001f)
+                {
+                    ++m_NonZeroLogCount;
+                }
+            }
+        }
+#endif
+
+#if UNITY_EDITOR || DEVELOPMENT_BUILD
+        private void StartDebugTestTone()
+        {
+            int sampleRate = AudioSettings.outputSampleRate > 0 ? AudioSettings.outputSampleRate : 48000;
+            int sampleCount = sampleRate;
+            float[] samples = new float[sampleCount];
+            for (int i = 0; i < sampleCount; ++i)
+            {
+                samples[i] = Mathf.Sin(2.0f * Mathf.PI * m_DebugTestToneFrequency * i / sampleRate);
+            }
+
+            AudioClip clip = AudioClip.Create("AppAudioMonitorDebugTone", sampleCount, 1,
+                sampleRate, false);
+            clip.SetData(samples, 0);
+            AudioSource debugTestToneSource = gameObject.AddComponent<AudioSource>();
+            debugTestToneSource.clip = clip;
+            debugTestToneSource.loop = true;
+            debugTestToneSource.volume = m_DebugTestToneVolume;
+            debugTestToneSource.spatialBlend = 0.0f;
+            debugTestToneSource.Play();
+#if UNITY_ANDROID
+            Debug.Log($"{kAndroidAppAudioLogPrefix} Debug test tone enabled frequency={m_DebugTestToneFrequency:F1} volume={m_DebugTestToneVolume:F3} sampleRate={sampleRate}");
+#endif
+        }
+#endif
     }
 
 }

Assets/Scripts/AudioCaptureManager.cs

@@ -43,6 +43,8 @@ namespace TiltBrush
 
     public class AudioCaptureManager : MonoBehaviour
     {
+        private const string kAndroidAppAudioLogPrefix = "AR_ANDROID_APP_AUDIO_20260603";
+
         // Number of seconds to delay before searching for active audio device.
         // From experimentation this seems to be the minimum to ensure we don't pick up
         // any residual audio.
@@ -75,9 +77,14 @@ private void ResetAudioCaptureType()
             m_Instance = this;
             if (LuaManager.Instance != null) LuaManager.Instance.VisualizerScriptingEnabled = false;
 #if UNITY_ANDROID || UNITY_IOS
+            // Mobile audio reactivity listens only to Unity's mixed app output.
+            // Mic, system audio, and other-app audio are separate unsupported sources.
             m_Type = AudioCaptureType.App;
 #else
             m_Type = AudioCaptureType.System;
+#endif
+#if UNITY_ANDROID
+            Debug.Log($"{kAndroidAppAudioLogPrefix} ResetAudioCaptureType selected {m_Type}");
 #endif
             m_CaptureRequestedCount = 0;
 
@@ -184,9 +191,9 @@ public string GetCaptureStatusMessage()
         {
             switch (m_Type)
             {
-                case AudioCaptureType.File: return "Listening to Mic'";
+                case AudioCaptureType.File: return "Listening to audio file";
                 case AudioCaptureType.System: return m_SystemAudio.GetCaptureStatusMessage();
-                case AudioCaptureType.App: return "Jammin'";
+                case AudioCaptureType.App: return "Listening to app audio";
                 case AudioCaptureType.Script: return "Scripted Waveform";
             }
             return "";
@@ -224,7 +231,15 @@ public void CaptureAudio(bool bCapture)
                     }
                     break;
                 case AudioCaptureType.App:
-                    m_AppAudio.SetActive(bCapture);
+                    bool appAudioWasActive = m_AppAudio.activeSelf;
+                    m_AppAudio.SetActive(CaptureRequested);
+                    if (appAudioWasActive != m_AppAudio.activeSelf)
+                    {
+                        VisualizerManager.m_Instance.AudioCaptureStatusChange(m_AppAudio.activeSelf);
+                    }
+#if UNITY_ANDROID
+                    Debug.Log($"{kAndroidAppAudioLogPrefix} CaptureAudio({bCapture}) set AppAudio active={m_AppAudio.activeSelf} requests={m_CaptureRequestedCount}");
+#endif
                     break;
                 case AudioCaptureType.Script:
                     break;

Assets/Scripts/SystemAudioMonitor.cs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
 using CSCore.CoreAudioAPI;
 using CSCore.SoundIn;
 using CSCore.Streams;
@@ -107,7 +107,7 @@ public void Clear()
         private float[] m_LChannelTempBuffer;
         private float[] m_RChannelTempBuffer;
 
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
         // Data that is only valid in State.Looking
         private Future<Queue<WasapiCapture>> m_CapturesFuture;
 
@@ -150,7 +150,7 @@ public int GetAudioDeviceSampleRate()
         void Awake()
         {
             m_State = State.Disabled;
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
             int size = VisualizerManager.m_Instance.FFTSize;
             m_HotValues = new StereoBuffer(size);
             m_OperateValues = new StereoBuffer(size);
@@ -179,7 +179,7 @@ public string GetCaptureStatusMessage()
             }
         }
 
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
 
         public void Activate(float delaySeconds)
         {

Assets/Scripts/Tools/MultiCamTool.cs

@@ -1618,7 +1618,11 @@ void UpdateAudioSearch()
                 else
                 {
                     m_VideoRecordAudioHeader.text = m_AudioLookingText;
+#if UNITY_ANDROID || UNITY_IOS
+                    m_VideoRecordAudioDesc.text = "Play audio in the app";
+#else
                     m_VideoRecordAudioDesc.text = "Play some sound or music on your computer";
+#endif
                 }
             }
             else if (m_AudioFoundCountdown > 0.0f)

Assets/Scripts/VisualizerCSCoreFFT.cs

@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
 using CSCore.DSP;
 #endif
 
@@ -20,7 +20,7 @@ namespace TiltBrush
     /// Wrapper for CSCore.DSP.FftProvider
     public class VisualizerCSCoreFft : VisualizerManager.Fft
     {
-#if !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX
+#if !DISABLE_SYSTEM_AUDIO_CAPTURE && !DISABLE_AUDIO_CAPTURE && !UNITY_OSX && !UNITY_EDITOR_OSX && !UNITY_ANDROID && !UNITY_IOS
         private FftProvider m_Fft;
         public VisualizerCSCoreFft(int channels, int fftSize)
         {