diff --git a/audioconverter.cpp b/audioconverter.cpp index 44bd25f..edc6ec7 100644 --- a/audioconverter.cpp +++ b/audioconverter.cpp @@ -82,224 +82,225 @@ audioConverter::~audioConverter() bool audioConverter::convert(audioPacket audio) { - if (audio.data.size() == 0) + // If inFormat and outFormat are identical, just emit the data back. + if (audio.data.size() != 0 && inFormat != outFormat) { - return false; - } - if (inFormat.codec() == "audio/opus") - { - unsigned char* in = (unsigned char*)audio.data.data(); + if (inFormat.codec() == "audio/opus") + { + unsigned char* in = (unsigned char*)audio.data.data(); - //Decode the frame. - int nSamples = opus_packet_get_nb_samples(in, audio.data.size(), inFormat.sampleRate()); - if (nSamples == -1) { - // No opus data yet? - return false; - } - QByteArray outPacket(nSamples * sizeof(float) * inFormat.channelCount(), (char)0xff); // Preset the output buffer size. - float* out = (float*)outPacket.data(); + //Decode the frame. + int nSamples = opus_packet_get_nb_samples(in, audio.data.size(), inFormat.sampleRate()); + if (nSamples == -1) { + // No opus data yet? + return false; + } + QByteArray outPacket(nSamples * sizeof(float) * inFormat.channelCount(), (char)0xff); // Preset the output buffer size. + float* out = (float*)outPacket.data(); - //if (audio.seq > lastAudioSequence + 1) { - // nSamples = opus_decode_float(opusDecoder, Q_NULLPTR, 0, out, nSamples, 1); - //} - //else { - nSamples = opus_decode_float(opusDecoder, in, audio.data.size(), out, nSamples, 0); - //} - //lastAudioSequence = audio.seq; - audio.data.clear(); - audio.data = outPacket; // Replace incoming data with converted. - } - else if (inFormat.codec() == "audio/PCMU") - { - // Current packet is "technically" 8bit so need to create a new buffer that is 16bit - QByteArray outPacket((int)audio.data.length() * 2, (char)0xff); - qint16* out = (qint16*)outPacket.data(); - for (int f = 0; f < audio.data.length(); f++) - { - *out++ = ulaw_decode[(quint8)audio.data[f]]; - } - audio.data.clear(); - audio.data = outPacket; // Replace incoming data with converted. - // Make sure that sample size/type is set correctly - } - - Eigen::VectorXf samplesF; + //if (audio.seq > lastAudioSequence + 1) { + // nSamples = opus_decode_float(opusDecoder, Q_NULLPTR, 0, out, nSamples, 1); + //} + //else { + nSamples = opus_decode_float(opusDecoder, in, audio.data.size(), out, nSamples, 0); + //} + //lastAudioSequence = audio.seq; + audio.data.clear(); + audio.data = outPacket; // Replace incoming data with converted. + } + else if (inFormat.codec() == "audio/PCMU") + { + // Current packet is "technically" 8bit so need to create a new buffer that is 16bit + QByteArray outPacket((int)audio.data.length() * 2, (char)0xff); + qint16* out = (qint16*)outPacket.data(); + for (int f = 0; f < audio.data.length(); f++) + { + *out++ = ulaw_decode[(quint8)audio.data[f]]; + } + audio.data.clear(); + audio.data = outPacket; // Replace incoming data with converted. + // Make sure that sample size/type is set correctly + } - if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 32) - { - Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint32))); - samplesF = samplesI.cast() / float(std::numeric_limits::max()); - } - else if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 16) - { - Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint16))); - samplesF = samplesI.cast() / float(std::numeric_limits::max()); - } - else if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 8) - { - Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint8))); - samplesF = samplesI.cast() / float(std::numeric_limits::max());; - } - else if (inFormat.sampleType() == QAudioFormat::UnSignedInt && inFormat.sampleSize() == 8) - { - Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(quint8))); - samplesF = samplesI.cast() / float(std::numeric_limits::max());; - } - else if (inFormat.sampleType() == QAudioFormat::Float) { - samplesF = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(float))); - } - else - { - qInfo(logAudio()) << "Unsupported Sample Type:" << inFormat.sampleType() << "Size:" << inFormat.sampleSize(); - } + Eigen::VectorXf samplesF; + + if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 32) + { + Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint32))); + samplesF = samplesI.cast() / float(std::numeric_limits::max()); + } + else if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 16) + { + Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint16))); + samplesF = samplesI.cast() / float(std::numeric_limits::max()); + } + else if (inFormat.sampleType() == QAudioFormat::SignedInt && inFormat.sampleSize() == 8) + { + Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(qint8))); + samplesF = samplesI.cast() / float(std::numeric_limits::max());; + } + else if (inFormat.sampleType() == QAudioFormat::UnSignedInt && inFormat.sampleSize() == 8) + { + Eigen::Ref samplesI = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(quint8))); + samplesF = samplesI.cast() / float(std::numeric_limits::max());; + } + else if (inFormat.sampleType() == QAudioFormat::Float) { + samplesF = Eigen::Map(reinterpret_cast(audio.data.data()), audio.data.size() / int(sizeof(float))); + } + else + { + qInfo(logAudio()) << "Unsupported Sample Type:" << inFormat.sampleType() << "Size:" << inFormat.sampleSize(); + } - audio.amplitude = samplesF.array().abs().maxCoeff(); - // Set the volume - samplesF *= audio.volume; + audio.amplitude = samplesF.array().abs().maxCoeff(); + // Set the volume + samplesF *= audio.volume; - /* - samplesF is now an Eigen Vector of the current samples in float format - The next step is to convert to the correct number of channels in outFormat.channelCount() - */ + /* + samplesF is now an Eigen Vector of the current samples in float format + The next step is to convert to the correct number of channels in outFormat.channelCount() + */ - if (inFormat.channelCount() == 2 && outFormat.channelCount() == 1) { - // If we need to drop one of the audio channels, do it now - Eigen::VectorXf samplesTemp(samplesF.size() / 2); - samplesTemp = Eigen::Map >(samplesF.data(), samplesF.size() / 2); - samplesF = samplesTemp; - } - else if (inFormat.channelCount() == 1 && outFormat.channelCount() == 2) { - // Convert mono to stereo if required - Eigen::VectorXf samplesTemp(samplesF.size() * 2); - Eigen::Map >(samplesTemp.data(), samplesF.size()) = samplesF; - Eigen::Map >(samplesTemp.data() + 1, samplesF.size()) = samplesF; - samplesF = samplesTemp; - } + if (inFormat.channelCount() == 2 && outFormat.channelCount() == 1) { + // If we need to drop one of the audio channels, do it now + Eigen::VectorXf samplesTemp(samplesF.size() / 2); + samplesTemp = Eigen::Map >(samplesF.data(), samplesF.size() / 2); + samplesF = samplesTemp; + } + else if (inFormat.channelCount() == 1 && outFormat.channelCount() == 2) { + // Convert mono to stereo if required + Eigen::VectorXf samplesTemp(samplesF.size() * 2); + Eigen::Map >(samplesTemp.data(), samplesF.size()) = samplesF; + Eigen::Map >(samplesTemp.data() + 1, samplesF.size()) = samplesF; + samplesF = samplesTemp; + } - /* - Next step is to resample (if needed) - */ + /* + Next step is to resample (if needed) + */ - if (resampler != Q_NULLPTR && resampleRatio != 1.0) - { - quint32 outFrames = ((samplesF.size() / outFormat.channelCount()) * resampleRatio); - quint32 inFrames = (samplesF.size() / outFormat.channelCount()); - QByteArray outPacket(outFrames * outFormat.channelCount() * sizeof(float), (char)0xff); // Preset the output buffer size. - const float* in = (float*)samplesF.data(); - float* out = (float*)outPacket.data(); + if (resampler != Q_NULLPTR && resampleRatio != 1.0) + { + quint32 outFrames = ((samplesF.size() / outFormat.channelCount()) * resampleRatio); + quint32 inFrames = (samplesF.size() / outFormat.channelCount()); + QByteArray outPacket(outFrames * outFormat.channelCount() * sizeof(float), (char)0xff); // Preset the output buffer size. + const float* in = (float*)samplesF.data(); + float* out = (float*)outPacket.data(); - int err = 0; - if (outFormat.channelCount() == 1) { - err = wf_resampler_process_float(resampler, 0, in, &inFrames, out, &outFrames); - } - else { - err = wf_resampler_process_interleaved_float(resampler, in, &inFrames, out, &outFrames); - } + int err = 0; + if (outFormat.channelCount() == 1) { + err = wf_resampler_process_float(resampler, 0, in, &inFrames, out, &outFrames); + } + else { + err = wf_resampler_process_interleaved_float(resampler, in, &inFrames, out, &outFrames); + } - if (err) { - qInfo(logAudioConverter()) << "Resampler error " << err << " inFrames:" << inFrames << " outFrames:" << outFrames; - } - samplesF = Eigen::Map(reinterpret_cast(outPacket.data()), outPacket.size() / int(sizeof(float))); - } + if (err) { + qInfo(logAudioConverter()) << "Resampler error " << err << " inFrames:" << inFrames << " outFrames:" << outFrames; + } + samplesF = Eigen::Map(reinterpret_cast(outPacket.data()), outPacket.size() / int(sizeof(float))); + } - /* - If output is Opus so encode it now, don't do any more conversion on the output of Opus. - */ + /* + If output is Opus so encode it now, don't do any more conversion on the output of Opus. + */ - if (outFormat.codec() == "audio/opus") - { - float* in = (float*)samplesF.data(); - QByteArray outPacket(1275, (char)0xff); // Preset the output buffer size to MAXIMUM possible Opus frame size - unsigned char* out = (unsigned char*)outPacket.data(); + if (outFormat.codec() == "audio/opus") + { + float* in = (float*)samplesF.data(); + QByteArray outPacket(1275, (char)0xff); // Preset the output buffer size to MAXIMUM possible Opus frame size + unsigned char* out = (unsigned char*)outPacket.data(); - int nbBytes = opus_encode_float(opusEncoder, in, (samplesF.size() / outFormat.channelCount()), out, outPacket.length()); - if (nbBytes < 0) - { - qInfo(logAudioConverter()) << "Opus encode failed:" << opus_strerror(nbBytes) << "Num Samples:" << samplesF.size(); - return false; - } - else { - outPacket.resize(nbBytes); - audio.data.clear(); - audio.data = outPacket; // Copy output packet back to input buffer. - //samplesF = Eigen::Map(reinterpret_cast(outPacket.data()), outPacket.size() / int(sizeof(float))); - } + int nbBytes = opus_encode_float(opusEncoder, in, (samplesF.size() / outFormat.channelCount()), out, outPacket.length()); + if (nbBytes < 0) + { + qInfo(logAudioConverter()) << "Opus encode failed:" << opus_strerror(nbBytes) << "Num Samples:" << samplesF.size(); + return false; + } + else { + outPacket.resize(nbBytes); + audio.data.clear(); + audio.data = outPacket; // Copy output packet back to input buffer. + //samplesF = Eigen::Map(reinterpret_cast(outPacket.data()), outPacket.size() / int(sizeof(float))); + } - } - else { + } + else { - /* - Now convert back into the output format required - */ - audio.data.clear(); + /* + Now convert back into the output format required + */ + audio.data.clear(); - if (outFormat.sampleType() == QAudioFormat::UnSignedInt && outFormat.sampleSize() == 8) - { - Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); - VectorXuint8 samplesI = samplesITemp.cast(); - audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(quint8))); - } - if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 8) - { - Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); - VectorXint8 samplesI = samplesITemp.cast(); - audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint8))); - } - if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 16) - { - Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); - VectorXint16 samplesI = samplesITemp.cast(); - audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint16))); - } - else if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 32) - { - Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); - VectorXint32 samplesI = samplesITemp.cast(); - audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint32))); - } - else if (outFormat.sampleType() == QAudioFormat::Float) - { - audio.data = QByteArray(reinterpret_cast(samplesF.data()), int(samplesF.size()) * int(sizeof(float))); - } - else { - qInfo(logAudio()) << "Unsupported Sample Type:" << outFormat.sampleType(); - } + if (outFormat.sampleType() == QAudioFormat::UnSignedInt && outFormat.sampleSize() == 8) + { + Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); + VectorXuint8 samplesI = samplesITemp.cast(); + audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(quint8))); + } + if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 8) + { + Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); + VectorXint8 samplesI = samplesITemp.cast(); + audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint8))); + } + if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 16) + { + Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); + VectorXint16 samplesI = samplesITemp.cast(); + audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint16))); + } + else if (outFormat.sampleType() == QAudioFormat::SignedInt && outFormat.sampleSize() == 32) + { + Eigen::VectorXf samplesITemp = samplesF * float(std::numeric_limits::max()); + VectorXint32 samplesI = samplesITemp.cast(); + audio.data = QByteArray(reinterpret_cast(samplesI.data()), int(samplesI.size()) * int(sizeof(qint32))); + } + else if (outFormat.sampleType() == QAudioFormat::Float) + { + audio.data = QByteArray(reinterpret_cast(samplesF.data()), int(samplesF.size()) * int(sizeof(float))); + } + else { + qInfo(logAudio()) << "Unsupported Sample Type:" << outFormat.sampleType(); + } - /* - As we currently don't have a float based uLaw encoder, this must be done - after all other conversion has taken place. - */ + /* + As we currently don't have a float based uLaw encoder, this must be done + after all other conversion has taken place. + */ - if (outFormat.codec() == "audio/PCMU") - { - QByteArray outPacket((int)audio.data.length() / 2, (char)0xff); - qint16* in = (qint16*)audio.data.data(); - for (int f = 0; f < outPacket.length(); f++) - { - qint16 sample = *in++; - int sign = (sample >> 8) & 0x80; - if (sign) - sample = (short)-sample; - if (sample > cClip) - sample = cClip; - sample = (short)(sample + cBias); - int exponent = (int)MuLawCompressTable[(sample >> 7) & 0xFF]; - int mantissa = (sample >> (exponent + 3)) & 0x0F; - int compressedByte = ~(sign | (exponent << 4) | mantissa); - outPacket[f] = (quint8)compressedByte; - } - audio.data.clear(); - audio.data = outPacket; // Copy output packet back to input buffer. + if (outFormat.codec() == "audio/PCMU") + { + QByteArray outPacket((int)audio.data.length() / 2, (char)0xff); + qint16* in = (qint16*)audio.data.data(); + for (int f = 0; f < outPacket.length(); f++) + { + qint16 sample = *in++; + int sign = (sample >> 8) & 0x80; + if (sign) + sample = (short)-sample; + if (sample > cClip) + sample = cClip; + sample = (short)(sample + cBias); + int exponent = (int)MuLawCompressTable[(sample >> 7) & 0xFF]; + int mantissa = (sample >> (exponent + 3)) & 0x0F; + int compressedByte = ~(sign | (exponent << 4) | mantissa); + outPacket[f] = (quint8)compressedByte; + } + audio.data.clear(); + audio.data = outPacket; // Copy output packet back to input buffer. + + } + } + } - } - } emit converted(audio); return true; } diff --git a/audioconverter.h b/audioconverter.h index 5a279b0..cd598b4 100644 --- a/audioconverter.h +++ b/audioconverter.h @@ -85,6 +85,7 @@ static inline QAudioFormat toQAudioFormat(quint8 codec, quint32 sampleRate) format.setSampleType(QAudioFormat::UnSignedInt); format.setByteOrder(QAudioFormat::LittleEndian); format.setCodec("audio/pcm"); + format.setSampleRate(sampleRate); if (codec == 0x01 || codec == 0x20) { /* Set sample to be what is expected by the encoder and the output of the decoder */ @@ -108,8 +109,7 @@ static inline QAudioFormat toQAudioFormat(quint8 codec, quint32 sampleRate) format.setCodec("audio/opus"); } - format.setSampleRate(sampleRate); return format; } -#endif \ No newline at end of file +#endif diff --git a/audiohandler.cpp b/audiohandler.cpp index aaf6b34..b4678e3 100644 --- a/audiohandler.cpp +++ b/audiohandler.cpp @@ -209,8 +209,8 @@ void audioHandler::setVolume(unsigned char volume) void audioHandler::incomingAudio(audioPacket packet) { - //QTime startProcessing = QTime::currentTime(); - if (audioDevice != Q_NULLPTR) { + + if (audioDevice != Q_NULLPTR && packet.data.size() > 0) { packet.volume = volume; emit sendToConverter(packet); @@ -221,32 +221,35 @@ void audioHandler::incomingAudio(audioPacket packet) void audioHandler::convertedOutput(audioPacket packet) { - currentLatency = packet.time.msecsTo(QTime::currentTime()) + (outFormat.durationForBytes(audioOutput->bufferSize() - audioOutput->bytesFree()) / 1000); - if (audioDevice != Q_NULLPTR) { - audioDevice->write(packet.data); - if (lastReceived.msecsTo(QTime::currentTime()) > 100) { - qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Time since last audio packet" << lastReceived.msecsTo(QTime::currentTime()) << "Expected around" << setup.blockSize; - } - lastReceived = QTime::currentTime(); - } - /*if ((packet.seq > lastSentSeq + 1) && (setup.codec == 0x40 || setup.codec == 0x80)) { - qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Attempting FEC on packet" << packet.seq << "as last is" << lastSentSeq; - lastSentSeq = packet.seq; - incomingAudio(packet); // Call myself again to run the packet a second time (FEC) - } - */ - lastSentSeq = packet.seq; - emit haveLevels(getAmplitude(), setup.latency, currentLatency, isUnderrun); + if (packet.data.size() > 0 ) { - amplitude = packet.amplitude; + currentLatency = packet.time.msecsTo(QTime::currentTime()) + (outFormat.durationForBytes(audioOutput->bufferSize() - audioOutput->bytesFree()) / 1000); + if (audioDevice != Q_NULLPTR) { + audioDevice->write(packet.data); + if (lastReceived.msecsTo(QTime::currentTime()) > 100) { + qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Time since last audio packet" << lastReceived.msecsTo(QTime::currentTime()) << "Expected around" << setup.blockSize; + } + lastReceived = QTime::currentTime(); + } + /*if ((packet.seq > lastSentSeq + 1) && (setup.codec == 0x40 || setup.codec == 0x80)) { + qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Attempting FEC on packet" << packet.seq << "as last is" << lastSentSeq; + lastSentSeq = packet.seq; + incomingAudio(packet); // Call myself again to run the packet a second time (FEC) + } + */ + lastSentSeq = packet.seq; + emit haveLevels(getAmplitude(), setup.latency, currentLatency, isUnderrun); + amplitude = packet.amplitude; + } } void audioHandler::getNextAudioChunk() { - tempBuf.data.append(audioDevice->readAll()); - - while (tempBuf.data.length() >= outFormat.bytesForDuration(setup.blockSize * 1000)) { + if (audioDevice) { + tempBuf.data.append(audioDevice->readAll()); + } + if (tempBuf.data.length() >= outFormat.bytesForDuration(setup.blockSize * 1000)) { audioPacket packet; packet.time = QTime::currentTime(); packet.sent = 0; @@ -259,20 +262,27 @@ void audioHandler::getNextAudioChunk() emit sendToConverter(packet); } - return; + /* If there is still enough data in the buffer, call myself again in 20ms */ + if (tempBuf.data.length() >= outFormat.bytesForDuration(setup.blockSize * 1000)) { + QTimer::singleShot(setup.blockSize, this, &audioHandler::getNextAudioChunk); + } + + return; } void audioHandler::convertedInput(audioPacket audio) { - emit haveAudioData(audio); - if (lastReceived.msecsTo(QTime::currentTime()) > 100) { - qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Time since last audio packet" << lastReceived.msecsTo(QTime::currentTime()) << "Expected around" << setup.blockSize ; - } - lastReceived = QTime::currentTime(); - amplitude = audio.amplitude; - emit haveLevels(getAmplitude(), setup.latency, currentLatency, isUnderrun); + if (audio.data.size() > 0) { + emit haveAudioData(audio); + if (lastReceived.msecsTo(QTime::currentTime()) > 100) { + qDebug(logAudio()) << (setup.isinput ? "Input" : "Output") << "Time since last audio packet" << lastReceived.msecsTo(QTime::currentTime()) << "Expected around" << setup.blockSize ; + } + lastReceived = QTime::currentTime(); + amplitude = audio.amplitude; + emit haveLevels(getAmplitude(), setup.latency, currentLatency, isUnderrun); + } } void audioHandler::changeLatency(const quint16 newSize)