From fc9e04ffd2adcc54bfab09d3eb1559aba693056c Mon Sep 17 00:00:00 2001 From: Lucas Schumacher Date: Thu, 9 May 2024 19:23:07 -0400 Subject: [PATCH] Almost working audio fft input for waterfall --- Cargo.lock | 321 ++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 + src/app.rs | 33 ++--- src/app/audio_fft.rs | 83 +++++++++++ src/app/waterfall.rs | 4 +- 5 files changed, 416 insertions(+), 28 deletions(-) create mode 100644 src/app/audio_fft.rs diff --git a/Cargo.lock b/Cargo.lock index 182e35f..ff4764b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,7 +73,7 @@ dependencies = [ "once_cell", "paste", "static_assertions", - "windows", + "windows 0.48.0", ] [[package]] @@ -117,6 +117,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fe60779335388a88c01ac6c3be40304d1e349de3ada3b15f7808bb90fa9dce" +dependencies = [ + "alsa-sys", + "bitflags 2.5.0", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.5.2" @@ -162,6 +183,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "anyhow" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" + [[package]] name = "arboard" version = "3.3.2" @@ -432,6 +459,26 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.60", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -594,6 +641,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -615,6 +671,17 @@ dependencies = [ "libc", ] +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clipboard-win" version = "5.3.1" @@ -719,6 +786,49 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f01585027057ff5f0a5bf276174ae4c1594a2c5bde93d5f46a016d76270f5a9" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -759,6 +869,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "derivative" version = "2.2.0" @@ -897,6 +1013,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" + [[package]] name = "emath" version = "0.27.2" @@ -1217,6 +1339,12 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "glow" version = "0.13.1" @@ -1402,6 +1530,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "jni" version = "0.21.1" @@ -1448,6 +1585,18 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.153" @@ -1509,6 +1658,15 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -1551,6 +1709,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -1610,6 +1774,45 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -1717,6 +1920,29 @@ dependencies = [ "objc", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -1871,6 +2097,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "primal-check" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9df7f93fd637f083201473dab4fee2db4c429d32e55e3299980ab3957ab916a0" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -1959,6 +2194,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cc3bcbdb1ddfc11e700e62968e6b4cc9c75bb466464ad28fb61c5b2c964418b" +[[package]] +name = "realfft" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953d9f7e5cdd80963547b456251296efc2626ed4e3cbf36c869d9564e0220571" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -2006,6 +2250,27 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.37.27" @@ -2109,6 +2374,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -2209,6 +2480,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -2377,6 +2654,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "ttf-parser" version = "0.20.0" @@ -2537,10 +2824,13 @@ name = "waterfall-glow-rs" version = "0.1.0" dependencies = [ "android_logger", + "anyhow", + "cpal", "eframe", "egui", "env_logger", "log", + "realfft", "wasm-bindgen-futures", "winit", ] @@ -2733,6 +3023,26 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result", + "windows-targets 0.52.5", +] + [[package]] name = "windows-implement" version = "0.48.0" @@ -2755,6 +3065,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "windows-result" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749f0da9cc72d82e600d8d2e44cadd0b9eedb9038f71a1c58556ac1c5791813b" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index 6f7b969..9cba2f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,11 @@ edition = "2021" # Common dependencies [dependencies] +anyhow = "1.0.83" +cpal = "0.15.3" egui = "0.27.0" log = "0.4.21" +realfft = "3.3.0" # eframe features for non android targets [target.'cfg(not(target_os = "android"))'.dependencies.eframe] diff --git a/src/app.rs b/src/app.rs index f6cad43..35768fa 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,26 +1,14 @@ use eframe::{egui_glow, glow}; use egui::mutex::Mutex; -use std::sync::mpsc; use std::sync::Arc; mod waterfall; use waterfall::Waterfall; +mod audio_fft; +use audio_fft::AudioFFT; pub mod turbo_colormap; -mod deadbeef_rand { - static mut RNG_SEED: u32 = 0x3d2faba7; - static mut RNG_BEEF: u32 = 0xdeadbeef; - pub fn rand() -> u8 { - unsafe { - RNG_SEED = (RNG_SEED << 7) ^ ((RNG_SEED >> 25).wrapping_add(RNG_BEEF)); - RNG_BEEF = (RNG_BEEF << 7) ^ ((RNG_BEEF >> 25).wrapping_add(0xdeadbeef)); - (RNG_SEED & 0xff) as u8 - } - } -} -use deadbeef_rand::rand; - -const WF_SIZE: usize = 1024; +const FFT_SIZE: usize = 1024; /// We derive Deserialize/Serialize so we can persist app state on shutdown. pub struct TemplateApp { @@ -29,7 +17,7 @@ pub struct TemplateApp { value: f32, /// Behind an `Arc>` so we can pass it to [`egui::PaintCallback`] and paint later. waterfall: Arc>, - fft_sender: mpsc::Sender>, + _stream: AudioFFT, } impl TemplateApp { @@ -41,7 +29,8 @@ impl TemplateApp { // Load previous app state (if any). // Note that you must enable the `persistence` feature for this to work. - let (tx, rx) = mpsc::channel(); + let (stream, rx) = AudioFFT::new(FFT_SIZE).unwrap(); + let wf_size = stream.output_len; let gl = cc .gl .as_ref() @@ -51,8 +40,8 @@ impl TemplateApp { // Example stuff: label: "Hello World!".to_owned(), value: 2.7, - waterfall: Arc::new(Mutex::new(Waterfall::new(gl, WF_SIZE, WF_SIZE, rx))), - fft_sender: tx, + waterfall: Arc::new(Mutex::new(Waterfall::new(gl, wf_size, wf_size, rx))), + _stream: stream, } } } @@ -127,12 +116,6 @@ impl eframe::App for TemplateApp { let _angle = response.drag_motion().x * 0.01; - let mut new_data = vec![0_u8; WF_SIZE]; - for data in new_data.iter_mut() { - *data = rand(); - } - self.fft_sender.send(new_data).unwrap(); - // Clone locals so we can move them into the paint callback: let waterfall = self.waterfall.clone(); diff --git a/src/app/audio_fft.rs b/src/app/audio_fft.rs new file mode 100644 index 0000000..172f2eb --- /dev/null +++ b/src/app/audio_fft.rs @@ -0,0 +1,83 @@ +use anyhow::{anyhow, Result}; +use cpal::{ + self, + traits::{DeviceTrait, HostTrait}, + BufferSize, StreamConfig, +}; +use realfft::RealFftPlanner; +use std::sync::mpsc; + +pub struct AudioFFT { + pub stream: cpal::Stream, + pub output_len: usize, +} + +impl AudioFFT { + pub fn new(size: usize) -> Result<(Self, mpsc::Receiver>)> { + let output_len = size / 2 + 1; + + // Create mpsc queue + let (tx, rx) = mpsc::channel(); + + // Setup fft use f32 for now + let mut fft_planner = RealFftPlanner::::new(); + let fft = fft_planner.plan_fft_forward(size); + + // Setup audio input + let host = cpal::default_host(); + let device = host + .default_input_device() + .ok_or(anyhow!("No input audio device found"))?; + // Basic config that 'should' be suppoted by most devices + let config = StreamConfig { + channels: 1, + sample_rate: cpal::SampleRate(44100), + buffer_size: BufferSize::Default, + }; + + let mut fft_in: Vec = Vec::with_capacity(size); + let mut fft_out = fft.make_output_vec(); + let mut fft_scratch = fft.make_scratch_vec(); + let stream = device.build_input_stream( + &config, + move |mut data: &[f32], _: &cpal::InputCallbackInfo| { + while data.fill_vec(&mut fft_in, size).is_ok() { + assert_eq!(size, fft_in.len()); + fft.process_with_scratch(&mut fft_in, &mut fft_out, &mut fft_scratch) + .unwrap(); + fft_in.clear(); + let output: Vec = fft_out.iter().map(|c| (c.arg() * 255.0) as u8).collect(); + assert_eq!(output_len, output.len()); + tx.send(output).unwrap(); + } + }, + move |err| log::error!("Audio Thread Error: {err}"), + None, + )?; + + Ok((Self { stream, output_len }, rx)) + } +} + +trait FillVec { + /// Takes elements from self and inserts them into out_vec + /// Returns Ok if out_vec is filled to size + /// Returns Err when out_vec is not fully filled (self will be empty) + fn fill_vec(&mut self, out_vec: &mut Vec, size: usize) -> Result<()>; +} +impl FillVec for &[f32] { + fn fill_vec(&mut self, out_vec: &mut Vec, size: usize) -> Result<()> { + let have = self.len(); + if have == 0 { + anyhow::bail!("Self empty"); + } + let need = size - out_vec.len(); + let can_move = need.min(have); + out_vec.extend_from_slice(&self[..can_move]); + *self = &self[can_move..]; + match out_vec.len() == size { + true => Ok(()), + false => Err(anyhow!("out_vec not full")), + } + } +} diff --git a/src/app/waterfall.rs b/src/app/waterfall.rs index 7ebccd6..76ce354 100644 --- a/src/app/waterfall.rs +++ b/src/app/waterfall.rs @@ -185,7 +185,6 @@ impl Waterfall { gl.tex_parameter_i32(TEXTURE_2D, TEXTURE_MAG_FILTER, NEAREST as i32); check_for_gl_errors(&gl, "Set texture params"); - //gl.tex_storage_2d(glow::TEXTURE_2D, 1, glow::R8, 300, 300); gl.tex_image_2d( glow::TEXTURE_2D, 0, @@ -195,7 +194,8 @@ impl Waterfall { 0, glow::RED, glow::UNSIGNED_BYTE, - Some(&buffer), + //Some(&buffer), // This segfaults with large buffers + None, ); check_for_gl_errors(&gl, "Initializing Texture");