#ifdef WITH_VOICE #ifdef USE_PORTAUDIO #include "manager.hpp" #include "abaddon.hpp" #include #include #include #include #include #include // TODO replace with libresample // a simple nearest-neigbhor resampler static void resampleBuffer(const int16_t *input, const int inputFrames, int16_t *output, const int outputFrames) { float ratio = (float) inputFrames / (float) outputFrames; inputFrames, outputFrames, ratio); for(int oi = 0; oi < outputFrames; oi++) { int ii = (int) (((float) oi) * ratio); // clamp the input index if(ii >= inputFrames) { ii = inputFrames - 1; } // copy the samples. * 2 because stereo output[oi * 2] = input[ii * 2]; output[oi * 2 + 1] = input[ii * 2 + 1]; } } int playCallback(const void *pInput, void *pOutput, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags, void *userData ) { AudioManager *mgr = reinterpret_cast(userData); if (mgr == nullptr) {return paContinue;}; std::lock_guard _(mgr->m_mutex); int16_t *resample_playback_buffer = mgr->resample_playback_buffer; auto *pOutputF32 = static_cast(resample_playback_buffer); /* clear the output buffer */ for(int i = 0; i<480 * 2; i++) pOutputF32[i] = 0; /* write every SSRC into the output stream */ for (auto &[ssrc, pair] : mgr->m_sources) { double volume = 1.0; if (const auto vol_it = mgr->m_volume_ssrc.find(ssrc); vol_it != mgr->m_volume_ssrc.end()) { volume = vol_it->second; } auto &buf = pair.first; const size_t n = std::min(static_cast(buf.size()), static_cast(480 * 2ULL)); for (size_t i = 0; i < n; i++) { pOutputF32[i] += (int16_t) (volume * buf[i]); //pOutputF32[i] += volume * buf[i] / 32768.F; } buf.erase(buf.begin(), buf.begin() + n); } /* scale the resample buffer to the output buffer */ resampleBuffer(resample_playback_buffer, 480, static_cast(pOutput), frameCount); return paContinue; } int recordCallback(const void *pInput, void *pOutput, unsigned long frameCount, const PaStreamCallbackTimeInfo* timeInfo, PaStreamCallbackFlags statusFlags, void *userData ) { auto *mgr = reinterpret_cast(userData); if (mgr == nullptr) return paContinue; int16_t *resample_capture_buffer = mgr->resample_capture_buffer; resampleBuffer(static_cast(pInput), frameCount, resample_capture_buffer, 480); mgr->OnCapturedPCM(static_cast(resample_capture_buffer), 480); /* * You can simply increment it by 480 in UDPSocket::SendEncrypted but this is wrong * The timestamp is supposed to be strictly linear eg. if there's discontinuous * transmission for 1 second then the timestamp should be 48000 greater than the * last packet. So it's incremented here because this is fired 100x per second * and is always called in sync with UDPSocket::SendEncrypted */ mgr->m_rtp_timestamp += 480; return paContinue; } AudioManager::AudioManager(const Glib::ustring &backends_string) : m_log(spdlog::stdout_color_mt("portaudio")) { m_ok = true; PaError pa_err; int opus_err; /* try to initialize PortAudio */ pa_err = Pa_Initialize(); if(pa_err != paNoError) { spdlog::get("audio")->error("ERROR: Pa_Initialize returned 0x%x\n", pa_err); Pa_Terminate(); m_ok = false; return; } /* initialize RNNoise */ #ifdef WITH_RNNOISE RNNoiseInitialize(); #endif /* try to initialize opus */ m_encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_VOIP, &opus_err); if (opus_err != OPUS_OK) { spdlog::get("audio")->error("failed to initialize opus encoder: {}", opus_err); m_ok = false; return; } opus_encoder_ctl(m_encoder, OPUS_SET_BITRATE(64000)); /* get the backends */ Enumerate(); /* create the configurations for capture and playback */ m_playback_config.sampleFormat = paInt16; m_playback_config.channelCount = 2; m_playback_config.suggestedLatency = Pa_GetDeviceInfo( m_capture_config.device )->defaultLowInputLatency; m_playback_config.hostApiSpecificStreamInfo = NULL; m_playback_config.device = Pa_GetDefaultOutputDevice(); resample_playback_source_rate = Pa_GetDeviceInfo( m_playback_config.device )->defaultSampleRate; resample_playback_framesPerBuffer = (int) (((float) resample_playback_source_rate / 48000.0) * 480.0); m_capture_config.sampleFormat = paInt16; m_capture_config.channelCount = 2; m_capture_config.suggestedLatency = Pa_GetDeviceInfo( m_capture_config.device )->defaultLowInputLatency; m_capture_config.hostApiSpecificStreamInfo = NULL; m_capture_config.device = Pa_GetDefaultInputDevice(); resample_capture_source_rate = Pa_GetDeviceInfo( m_capture_config.device )->defaultSampleRate; resample_capture_framesPerBuffer = (int) (((float) resample_capture_source_rate / 48000.0) * 480.0); /* initialize the buffers for resampling */ resample_capture_buffer = (int16_t*) malloc(480 * 2 * sizeof(uint16_t)); resample_playback_buffer = (int16_t*) malloc(480 * 2 * sizeof(uint16_t)); /* create the streams */ pa_err = Pa_OpenStream(&pa_capture_device, &m_capture_config, NULL, resample_capture_source_rate, resample_capture_framesPerBuffer, paClipOff, recordCallback, this); if(pa_err != paNoError) { m_ok = false; spdlog::get("audio")->error("ERROR: Pa_OpenStream for capture returned %d\n", pa_err); return; } pa_err = Pa_OpenStream(&pa_playback_device, NULL, &m_playback_config, resample_playback_source_rate, resample_playback_framesPerBuffer, paClipOff, playCallback, this); if(pa_err != paNoError) { m_ok = false; spdlog::get("audio")->error("ERROR: Pa_OpenStream for playback returned %d\n", pa_err); return; } if(Pa_StartStream(pa_playback_device) != paNoError) { spdlog::get("audio")->error("ERROR: Could not start playback\n"); } /**TODO if (const auto capture_id = m_devices.GetDefaultCapture(); capture_id.has_value()) { m_capture_id = *capture_id; m_capture_config.capture.pDeviceID = &m_capture_id; }*/ Glib::signal_timeout().connect(sigc::mem_fun(*this, &AudioManager::DecayVolumeMeters), 40); } AudioManager::~AudioManager() { Pa_Terminate(); RemoveAllSSRCs(); free(resample_capture_buffer); free(resample_playback_buffer); #ifdef WITH_RNNOISE RNNoiseUninitialize(); #endif } void AudioManager::AddSSRC(uint32_t ssrc) { std::lock_guard _(m_mutex); int error; if (m_sources.find(ssrc) == m_sources.end()) { auto *decoder = opus_decoder_create(48000, 2, &error); m_sources.insert(std::make_pair(ssrc, std::make_pair(std::deque {}, decoder))); } } void AudioManager::RemoveSSRC(uint32_t ssrc) { std::lock_guard _(m_mutex); if (auto it = m_sources.find(ssrc); it != m_sources.end()) { opus_decoder_destroy(it->second.second); m_sources.erase(it); } } void AudioManager::RemoveAllSSRCs() { spdlog::get("audio")->info("removing all ssrc"); std::lock_guard _(m_mutex); for (auto &[ssrc, pair] : m_sources) { opus_decoder_destroy(pair.second); } m_sources.clear(); } void AudioManager::SetOpusBuffer(uint8_t *ptr) { m_opus_buffer = ptr; } void AudioManager::FeedMeOpus(uint32_t ssrc, const std::vector &data) { if (!m_should_playback || Pa_IsStreamActive(pa_playback_device) != 1) return; std::lock_guard _(m_mutex); if (m_muted_ssrcs.find(ssrc) != m_muted_ssrcs.end()) return; static std::array pcm; if (auto it = m_sources.find(ssrc); it != m_sources.end()) { int decoded = opus_decode(it->second.second, data.data(), static_cast(data.size()), pcm.data(), 120 * 48, 0); if (decoded <= 0) { } else { UpdateReceiveVolume(ssrc, pcm.data(), decoded); auto &buf = it->second.first; buf.insert(buf.end(), pcm.begin(), pcm.begin() + decoded * 2); } } } void AudioManager::StartCaptureDevice() { if (Pa_StartStream(pa_capture_device) != paNoError) { spdlog::get("audio")->error("Failed to start capture device"); } } void AudioManager::StopCaptureDevice() { if (Pa_StopStream(pa_capture_device) != paNoError) { spdlog::get("audio")->error("Failed to stop capture device"); } } void AudioManager::SetPlaybackDevice(const Gtk::TreeModel::iterator &iter) { PaError pa_err; spdlog::get("audio")->debug("Setting new playback device"); const auto device_id = m_devices.GetPlaybackDeviceIDFromModel(iter); if (!device_id) { spdlog::get("audio")->error("Requested ID from iterator is invalid"); return; } m_devices.SetActivePlaybackDevice(iter); // TODO m_playback_config.device = *device_id; Pa_AbortStream(pa_playback_device); pa_err = Pa_OpenStream(&pa_playback_device, NULL, &m_playback_config, 48000, 480, paClipOff, playCallback, this); if(pa_err != paNoError) { spdlog::get("audio")->error("Error setting playback device"); } Pa_StartStream(pa_playback_device); } void AudioManager::SetCaptureDevice(const Gtk::TreeModel::iterator &iter) { PaError pa_err; spdlog::get("audio")->debug("Setting new capture device"); const auto device_id = m_devices.GetCaptureDeviceIDFromModel(iter); if (!device_id) { spdlog::get("audio")->error("Requested ID from iterator is invalid"); return; } m_devices.SetActiveCaptureDevice(iter); //TODO m_playback_config.device = *device_id; Pa_AbortStream(pa_playback_device); pa_err = Pa_OpenStream(&pa_capture_device, &m_capture_config, NULL, 48000, 480, paClipOff, recordCallback, this); Pa_StartStream(pa_playback_device); } void AudioManager::SetCapture(bool capture) { m_should_capture = capture; } void AudioManager::SetPlayback(bool playback) { m_should_playback = playback; } void AudioManager::SetCaptureGate(double gate) { m_capture_gate = gate; } void AudioManager::SetCaptureGain(double gain) { m_capture_gain = gain; } double AudioManager::GetCaptureGate() const noexcept { return m_capture_gate; } double AudioManager::GetCaptureGain() const noexcept { return m_capture_gain; } void AudioManager::SetMuteSSRC(uint32_t ssrc, bool mute) { std::lock_guard _(m_mutex); if (mute) { m_muted_ssrcs.insert(ssrc); } else { m_muted_ssrcs.erase(ssrc); } } void AudioManager::SetVolumeSSRC(uint32_t ssrc, double volume) { std::lock_guard _(m_mutex); m_volume_ssrc[ssrc] = volume; } double AudioManager::GetVolumeSSRC(uint32_t ssrc) const { std::lock_guard _(m_mutex); if (const auto iter = m_volume_ssrc.find(ssrc); iter != m_volume_ssrc.end()) { return iter->second; } return 1.0; } void AudioManager::SetEncodingApplication(int application) { std::lock_guard _(m_enc_mutex); int prev_bitrate = 64000; if (int err = opus_encoder_ctl(m_encoder, OPUS_GET_BITRATE(&prev_bitrate)); err != OPUS_OK) { spdlog::get("audio")->error("Failed to get old bitrate when reinitializing: {}", err); } opus_encoder_destroy(m_encoder); int err = 0; m_encoder = opus_encoder_create(48000, 2, application, &err); if (err != OPUS_OK) { spdlog::get("audio")->critical("opus_encoder_create failed: {}", err); return; } if (int err = opus_encoder_ctl(m_encoder, OPUS_SET_BITRATE(prev_bitrate)); err != OPUS_OK) { spdlog::get("audio")->error("Failed to set bitrate when reinitializing: {}", err); } } int AudioManager::GetEncodingApplication() { std::lock_guard _(m_enc_mutex); int temp = OPUS_APPLICATION_VOIP; if (int err = opus_encoder_ctl(m_encoder, OPUS_GET_APPLICATION(&temp)); err != OPUS_OK) { spdlog::get("audio")->error("opus_encoder_ctl(OPUS_GET_APPLICATION) failed: {}", err); } return temp; } void AudioManager::SetSignalHint(int signal) { std::lock_guard _(m_enc_mutex); if (int err = opus_encoder_ctl(m_encoder, OPUS_SET_SIGNAL(signal)); err != OPUS_OK) { spdlog::get("audio")->error("opus_encoder_ctl(OPUS_SET_SIGNAL) failed: {}", err); } } int AudioManager::GetSignalHint() { std::lock_guard _(m_enc_mutex); int temp = OPUS_AUTO; if (int err = opus_encoder_ctl(m_encoder, OPUS_GET_SIGNAL(&temp)); err != OPUS_OK) { spdlog::get("audio")->error("opus_encoder_ctl(OPUS_GET_SIGNAL) failed: {}", err); } return temp; } void AudioManager::SetBitrate(int bitrate) { std::lock_guard _(m_enc_mutex); if (int err = opus_encoder_ctl(m_encoder, OPUS_SET_BITRATE(bitrate)); err != OPUS_OK) { spdlog::get("audio")->error("opus_encoder_ctl(OPUS_SET_BITRATE) failed: {}", err); } } int AudioManager::GetBitrate() { std::lock_guard _(m_enc_mutex); int temp = 64000; if (int err = opus_encoder_ctl(m_encoder, OPUS_GET_BITRATE(&temp)); err != OPUS_OK) { spdlog::get("audio")->error("opus_encoder_ctl(OPUS_GET_BITRATE) failed: {}", err); } return temp; } void AudioManager::Enumerate() { spdlog::get("audio")->debug("Enumerating devices"); m_devices.SetDevices(); } void AudioManager::OnCapturedPCM(const int16_t *pcm, uint32_t frames) { if (m_opus_buffer == nullptr || !m_should_capture) return; const double gain = m_capture_gain; std::vector new_pcm(pcm, pcm + frames * 2); for (auto &val : new_pcm) { const int32_t unclamped = static_cast(val * gain); val = std::clamp(unclamped, INT16_MIN, INT16_MAX); } if (m_mix_mono) { for (size_t i = 0; i < frames * 2; i += 2) { const int sample_L = new_pcm[i]; const int sample_R = new_pcm[i + 1]; const int16_t mixed = static_cast((sample_L + sample_R) / 2); new_pcm[i] = mixed; new_pcm[i + 1] = mixed; } } UpdateCaptureVolume(new_pcm.data(), frames); static std::array denoised_L; static std::array denoised_R; bool m_rnnoise_passed = false; #ifdef WITH_RNNOISE if (m_vad_method == VADMethod::RNNoise || m_enable_noise_suppression) { m_rnnoise_passed = CheckVADRNNoise(new_pcm.data(), denoised_L.data(), denoised_R.data()); } #endif switch (m_vad_method) { case VADMethod::Gate: if (!CheckVADVoiceGate()) return; break; #ifdef WITH_RNNOISE case VADMethod::RNNoise: if (!m_rnnoise_passed) return; break; #endif } m_enc_mutex.lock(); int payload_len = -1; if (m_enable_noise_suppression) { static std::array denoised_interleaved; for (size_t i = 0; i < 480; i++) { denoised_interleaved[i * 2] = static_cast(denoised_L[i]); } for (size_t i = 0; i < 480; i++) { denoised_interleaved[i * 2 + 1] = static_cast(denoised_R[i]); } payload_len = opus_encode(m_encoder, denoised_interleaved.data(), 480, static_cast(m_opus_buffer), 1275); } else { payload_len = opus_encode(m_encoder, new_pcm.data(), 480, static_cast(m_opus_buffer), 1275); } m_enc_mutex.unlock(); if (payload_len < 0) { spdlog::get("audio")->error("encoding error: {}", payload_len); } else { m_signal_opus_packet.emit(payload_len); } } void AudioManager::UpdateReceiveVolume(uint32_t ssrc, const int16_t *pcm, int frames) { std::lock_guard _(m_vol_mtx); auto &meter = m_volumes[ssrc]; for (int i = 0; i < frames * 2; i += 2) { const int amp = std::abs(pcm[i]); meter = std::max(meter, std::abs(amp) / 32768.0); } } void AudioManager::UpdateCaptureVolume(const int16_t *pcm, uint32_t frames) { for (uint32_t i = 0; i < frames * 2; i += 2) { const int amp = std::abs(pcm[i]); m_capture_peak_meter = std::max(m_capture_peak_meter.load(std::memory_order_relaxed), amp); } } bool AudioManager::DecayVolumeMeters() { m_capture_peak_meter -= 600; if (m_capture_peak_meter < 0) m_capture_peak_meter = 0; const auto x = m_vad_prob.load() - 0.05f; m_vad_prob.store(x < 0.0f ? 0.0f : x); std::lock_guard _(m_vol_mtx); for (auto &[ssrc, meter] : m_volumes) { meter -= 0.01; if (meter < 0.0) meter = 0.0; } return true; } bool AudioManager::CheckVADVoiceGate() { return m_capture_peak_meter / 32768.0 > m_capture_gate; } #ifdef WITH_RNNOISE bool AudioManager::CheckVADRNNoise(const int16_t *pcm, float *denoised_left, float *denoised_right) { // use left channel for vad, only denoise right if noise suppression enabled std::unique_lock _(m_rnn_mutex); static float rnnoise_input[480]; for (size_t i = 0; i < 480; i++) { rnnoise_input[i] = static_cast(pcm[i * 2]); } m_vad_prob = std::max(m_vad_prob.load(), rnnoise_process_frame(m_rnnoise[0], denoised_left, rnnoise_input)); if (m_enable_noise_suppression) { for (size_t i = 0; i < 480; i++) { rnnoise_input[i] = static_cast(pcm[i * 2 + 1]); } rnnoise_process_frame(m_rnnoise[1], denoised_right, rnnoise_input); } return m_vad_prob > m_prob_threshold; } void AudioManager::RNNoiseInitialize() { spdlog::get("audio")->debug("Initializing RNNoise"); RNNoiseUninitialize(); std::unique_lock _(m_rnn_mutex); m_rnnoise[0] = rnnoise_create(nullptr); m_rnnoise[1] = rnnoise_create(nullptr); const auto expected = rnnoise_get_frame_size(); if (expected != 480) { spdlog::get("audio")->warn("RNNoise expects a frame count other than 480"); } } void AudioManager::RNNoiseUninitialize() { if (m_rnnoise[0] != nullptr) { spdlog::get("audio")->debug("Uninitializing RNNoise"); std::unique_lock _(m_rnn_mutex); rnnoise_destroy(m_rnnoise[0]); rnnoise_destroy(m_rnnoise[1]); m_rnnoise[0] = nullptr; m_rnnoise[1] = nullptr; } } #endif bool AudioManager::OK() const { return m_ok; } double AudioManager::GetCaptureVolumeLevel() const noexcept { return m_capture_peak_meter / 32768.0; } double AudioManager::GetSSRCVolumeLevel(uint32_t ssrc) const noexcept { std::lock_guard _(m_vol_mtx); if (const auto it = m_volumes.find(ssrc); it != m_volumes.end()) { return it->second; } return 0.0; } AudioDevices &AudioManager::GetDevices() { return m_devices; } uint32_t AudioManager::GetRTPTimestamp() const noexcept { return m_rtp_timestamp; } void AudioManager::SetVADMethod(const std::string &method) { spdlog::get("audio")->debug("Setting VAD method to {}", method); if (method == "gate") { SetVADMethod(VADMethod::Gate); } else if (method == "rnnoise") { #ifdef WITH_RNNOISE SetVADMethod(VADMethod::RNNoise); #else SetVADMethod(VADMethod::Gate); spdlog::get("audio")->error("Tried to set RNNoise VAD method with support disabled"); #endif } else { SetVADMethod(VADMethod::Gate); spdlog::get("audio")->error("Tried to set unknown VAD method {}", method); } } void AudioManager::SetVADMethod(VADMethod method) { const auto method_int = static_cast(method); spdlog::get("audio")->debug("Setting VAD method to enum {}", method_int); m_vad_method = method; } AudioManager::VADMethod AudioManager::GetVADMethod() const { return m_vad_method; } #ifdef WITH_RNNOISE float AudioManager::GetCurrentVADProbability() const { return m_vad_prob; } double AudioManager::GetRNNProbThreshold() const { return m_prob_threshold; } void AudioManager::SetRNNProbThreshold(double value) { m_prob_threshold = value; } void AudioManager::SetSuppressNoise(bool value) { m_enable_noise_suppression = value; } bool AudioManager::GetSuppressNoise() const { return m_enable_noise_suppression; } #endif void AudioManager::SetMixMono(bool value) { m_mix_mono = value; } bool AudioManager::GetMixMono() const { return m_mix_mono; } AudioManager::type_signal_opus_packet AudioManager::signal_opus_packet() { return m_signal_opus_packet; } #endif /* USE_PORTAUDIO */ #endif /* WITH_VOICE */