Skip to content

Commit 6e22771

Browse files
cchampetcchampet
authored
Merge pull request #271 from cchampet/dev_transcoderUseComplexFilterToMapAudio
Transcoder: use complex filter to map several audio
2 parents dfe757e + 2e61259 commit 6e22771

22 files changed

+571
-331
lines changed

‎src/AvTranscoder/data/decoded/Frame.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@ Frame::Frame(const Frame& otherFrame)
2323
refFrame(otherFrame);
2424
}
2525

26+
void Frame::operator=(const Frame& otherFrame)
27+
{
28+
// check if the frame could be a valid video/audio frame
29+
if(otherFrame.getAVFrame().format == -1)
30+
return;
31+
// reference the other frame
32+
refFrame(otherFrame);
33+
}
34+
2635
Frame::~Frame()
2736
{
2837
if(_frame != NULL)

‎src/AvTranscoder/data/decoded/Frame.hpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,11 @@ class AvExport Frame
2222
*/
2323
Frame();
2424

25-
/**
26-
* @brief Copy properties and reference data of the other frame
27-
*/
25+
//@{
26+
// @brief Copy properties and reference data of the other frame.
2827
Frame(const Frame& otherFrame);
28+
void operator=(const Frame& otherFrame);
29+
//@}
2930

3031
virtual ~Frame();
3132

@@ -63,7 +64,8 @@ class AvExport Frame
6364

6465
/**
6566
* @brief Copy frame properties and create a new reference to data of the given frame.
66-
* @warning This method allocates new data that will be freed only by calling the destructor of the referenced frame.
67+
* @warning This method allocates new data that will be freed by calling clear method or the destructor of the referenced frame.
68+
* @see clear
6769
*/
6870
void refFrame(const Frame& otherFrame);
6971

‎src/AvTranscoder/decoder/AudioDecoder.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ void AudioDecoder::setupDecoder(const ProfileLoader::Profile& profile)
7878
bool AudioDecoder::decodeNextFrame(Frame& frameBuffer)
7979
{
8080
bool decodeNextFrame = false;
81+
const size_t channelLayout = frameBuffer.getAVFrame().channel_layout;
8182

8283
if(!_isSetup)
8384
setupDecoder();
@@ -100,6 +101,9 @@ bool AudioDecoder::decodeNextFrame(Frame& frameBuffer)
100101
throw std::runtime_error("An error occurred during audio decoding: " + getDescriptionFromErrorCode(ret));
101102
}
102103

104+
// fixed channel layout value after decoding
105+
frameBuffer.getAVFrame().channel_layout = channelLayout;
106+
103107
// if no frame could be decompressed
104108
if(!nextPacketRead && ret == 0 && got_frame == 0)
105109
decodeNextFrame = false;

‎src/AvTranscoder/decoder/AudioGenerator.cpp

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,31 +22,20 @@ AudioGenerator::~AudioGenerator()
2222
bool AudioGenerator::decodeNextFrame(Frame& frameBuffer)
2323
{
2424
// check the given frame
25-
if(!frameBuffer.isAudioFrame())
25+
if(! frameBuffer.isAudioFrame())
2626
{
27-
LOG_WARN("The given frame is not a valid audio frame: allocate a new AVSample to put generated data into it.");
27+
LOG_WARN("The given frame to put data is not a valid audio frame: try to reallocate it.")
2828
frameBuffer.clear();
2929
static_cast<AudioFrame&>(frameBuffer).allocateAVSample(_frameDesc);
3030
}
3131

32-
// Check channel layout of the given frame to be able to copy audio data to it.
33-
// @see Frame.copyData method
34-
if(frameBuffer.getAVFrame().channel_layout == 0)
35-
{
36-
const size_t channelLayout = av_get_default_channel_layout(frameBuffer.getAVFrame().channels);
37-
LOG_WARN("Channel layout en the audio frame is not set. Set it to '" << channelLayout
38-
<< "' to be able to copy silence data.")
39-
av_frame_set_channel_layout(&frameBuffer.getAVFrame(), channelLayout);
40-
}
41-
4232
// Generate silent
4333
if(!_inputFrame)
4434
{
4535
// Generate the silent only once
4636
if(!_silent)
4737
{
48-
AudioFrame& audioBuffer = static_cast<AudioFrame&>(frameBuffer);
49-
_silent = new AudioFrame(audioBuffer.desc());
38+
_silent = new AudioFrame(_frameDesc);
5039

5140
std::stringstream msg;
5241
msg << "Generate a silence with the following features:" << std::endl;
@@ -61,7 +50,7 @@ bool AudioGenerator::decodeNextFrame(Frame& frameBuffer)
6150
_silent->setNbSamplesPerChannel(frameBuffer.getAVFrame().nb_samples);
6251
}
6352
LOG_DEBUG("Copy data of the silence when decode next frame")
64-
frameBuffer.copyData(*_silent);
53+
frameBuffer.refFrame(*_silent);
6554
}
6655
// Take audio frame from _inputFrame
6756
else

‎src/AvTranscoder/decoder/AudioGenerator.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class AvExport AudioGenerator : public IDecoder
2626
private:
2727
Frame* _inputFrame; ///< Has link (no ownership)
2828
AudioFrame* _silent; ///< The generated silent (has ownership)
29-
const AudioFrameDesc _frameDesc; ///< The description of the silence (sampleRate, channels...)
29+
const AudioFrameDesc _frameDesc; ///< The description of the given frame buffer when decoding.
3030
};
3131
}
3232

‎src/AvTranscoder/decoder/VideoGenerator.cpp

Lines changed: 14 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#include "VideoGenerator.hpp"
22

33
#include <AvTranscoder/util.hpp>
4-
#include <AvTranscoder/transform/VideoTransform.hpp>
54

65
#include <sstream>
76

@@ -12,6 +11,7 @@ VideoGenerator::VideoGenerator(const VideoFrameDesc& frameDesc)
1211
: _inputFrame(NULL)
1312
, _blackImage(NULL)
1413
, _frameDesc(frameDesc)
14+
, _videoTransform()
1515
{
1616
}
1717

@@ -23,9 +23,9 @@ VideoGenerator::~VideoGenerator()
2323
bool VideoGenerator::decodeNextFrame(Frame& frameBuffer)
2424
{
2525
// check the given frame
26-
if(!frameBuffer.isVideoFrame())
26+
if(! frameBuffer.isVideoFrame())
2727
{
28-
LOG_WARN("The given frame is not a valid video frame: allocate a new AVPicture to put generated data into it.");
28+
LOG_WARN("The given frame to put data is not a valid video frame: try to reallocate it.")
2929
frameBuffer.clear();
3030
static_cast<VideoFrame&>(frameBuffer).allocateAVPicture(_frameDesc);
3131
}
@@ -36,32 +36,23 @@ bool VideoGenerator::decodeNextFrame(Frame& frameBuffer)
3636
// Generate the black image only once
3737
if(!_blackImage)
3838
{
39+
// Create the black RGB image
40+
VideoFrameDesc blackDesc(_frameDesc._width, _frameDesc._height, "rgb24");
41+
_blackImage = new VideoFrame(blackDesc);
42+
const unsigned char fillChar = 0;
43+
_blackImage->assign(fillChar);
44+
3945
std::stringstream msg;
4046
msg << "Generate a black image with the following features:" << std::endl;
41-
msg << "width = " << _frameDesc._width << std::endl;
42-
msg << "height = " << _frameDesc._height << std::endl;
43-
msg << "pixel format = rgb24" << std::endl;
47+
msg << "width = " << _blackImage->getWidth() << std::endl;
48+
msg << "height = " << _blackImage->getHeight() << std::endl;
49+
msg << "pixel format = " << getPixelFormatName(_blackImage->getPixelFormat()) << std::endl;
4450
LOG_INFO(msg.str())
4551

46-
VideoFrame& imageBuffer = static_cast<VideoFrame&>(frameBuffer);
47-
48-
// Input of convert
49-
// @todo support PAL (0 to 255) and NTFS (16 to 235)
50-
VideoFrameDesc desc(_frameDesc);
51-
desc._pixelFormat = getAVPixelFormat("rgb24");
52-
VideoFrame intermediateBuffer(desc);
53-
const unsigned char fillChar = 0;
54-
intermediateBuffer.assign(fillChar);
55-
56-
// Output of convert
57-
_blackImage = new VideoFrame(imageBuffer.desc());
58-
59-
// Convert and store the black image
60-
VideoTransform videoTransform;
61-
videoTransform.convert(intermediateBuffer, *_blackImage);
6252
}
6353
LOG_DEBUG("Copy data of the black image when decode next frame")
64-
frameBuffer.copyData(*_blackImage);
54+
// Convert the black image to the configuration of the given frame
55+
_videoTransform.convert(*_blackImage, frameBuffer);
6556
}
6657
// Take image from _inputFrame
6758
else

‎src/AvTranscoder/decoder/VideoGenerator.hpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#include "IDecoder.hpp"
55
#include <AvTranscoder/codec/VideoCodec.hpp>
6+
#include <AvTranscoder/transform/VideoTransform.hpp>
67

78
namespace avtranscoder
89
{
@@ -15,18 +16,22 @@ class AvExport VideoGenerator : public IDecoder
1516

1617
public:
1718
VideoGenerator(const VideoFrameDesc& frameDesc);
18-
1919
~VideoGenerator();
2020

2121
bool decodeNextFrame(Frame& frameBuffer);
2222
bool decodeNextFrame(Frame& frameBuffer, const std::vector<size_t> channelIndexArray);
2323

24+
/**
25+
* @brief Force to return this frame when calling the decoding methods.
26+
* @param inputFrame: should have the same properties as the given frames when decoding.
27+
*/
2428
void setNextFrame(Frame& inputFrame) { _inputFrame = &inputFrame; }
2529

2630
private:
2731
Frame* _inputFrame; ///< A frame given from outside (has link, no ownership)
28-
VideoFrame* _blackImage; ///< The generated black image (has ownership)
29-
const VideoFrameDesc _frameDesc; ///< The description of the black image (width, height...)
32+
VideoFrame* _blackImage; ///< The generated RGB black image (has ownership)
33+
const VideoFrameDesc _frameDesc; ///< The description of the given frame buffer when decoding.
34+
VideoTransform _videoTransform; ///< To transform data of the back image to the given Frame when decoding.
3035
};
3136
}
3237

‎src/AvTranscoder/filter/FilterGraph.cpp

Lines changed: 68 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -35,30 +35,35 @@ FilterGraph::~FilterGraph()
3535
avfilter_graph_free(&_graph);
3636
}
3737

38-
void FilterGraph::process(Frame& frame)
38+
void FilterGraph::process(const std::vector<Frame*>& inputs, Frame& output)
3939
{
4040
if(!hasFilters())
4141
{
42-
LOG_DEBUG("No filter to process.")
42+
LOG_DEBUG("No filter to process: reference first input frame to the given output.")
43+
output.clear();
44+
output.refFrame(*inputs.at(0));
4345
return;
4446
}
4547

4648
// init filter graph
4749
if(!_isInit)
48-
init(frame);
50+
init(inputs, output);
4951

50-
// setup source frame
51-
int ret = av_buffersrc_write_frame(_filters.at(0)->getAVFilterContext(), &frame.getAVFrame());
52-
if(ret < 0)
52+
// setup input frames
53+
for(size_t index = 0; index < inputs.size(); ++index)
5354
{
54-
throw std::runtime_error("Error when adding a frame to the source buffer used to start to process filters: " +
55-
getDescriptionFromErrorCode(ret));
55+
const int ret = av_buffersrc_write_frame(_filters.at(index)->getAVFilterContext(), &inputs.at(index)->getAVFrame());
56+
if(ret < 0)
57+
{
58+
throw std::runtime_error("Error when adding a frame to the source buffer used to start to process filters: " +
59+
getDescriptionFromErrorCode(ret));
60+
}
5661
}
5762

5863
// pull filtered data from the filter graph
5964
for(;;)
6065
{
61-
ret = av_buffersink_get_frame(_filters.at(_filters.size() - 1)->getAVFilterContext(), &frame.getAVFrame());
66+
const int ret = av_buffersink_get_frame(_filters.at(_filters.size() - 1)->getAVFilterContext(), &output.getAVFrame());
6267
if(ret == AVERROR_EOF || ret == AVERROR(EAGAIN))
6368
break;
6469
if(ret < 0)
@@ -77,22 +82,31 @@ Filter& FilterGraph::addFilter(const std::string& filterName, const std::string&
7782
return *_filters.back();
7883
}
7984

80-
void FilterGraph::init(const Frame& frame)
85+
void FilterGraph::init(const std::vector<Frame*>& inputs, Frame& output)
8186
{
8287
// push filters to the graph
83-
pushInBuffer(frame);
84-
for(size_t i = 1; i < _filters.size(); ++i)
88+
addInBuffer(inputs);
89+
addOutBuffer(output);
90+
for(size_t i = 0; i < _filters.size(); ++i)
8591
{
8692
pushFilter(*_filters.at(i));
8793
}
88-
pushOutBuffer(frame);
8994

9095
// connect filters
9196
for(size_t index = 0; index < _filters.size() - 1; ++index)
9297
{
93-
LOG_INFO("Connect filter " << _filters.at(index)->getName() << " to filter " << _filters.at(index + 1)->getName())
98+
size_t indexOfOutputFilterToConnect = index + 1;
99+
size_t indexOfInputPadOfDestinationFilter = 0;
100+
// handle cases with several inputs
101+
if(index < inputs.size())
102+
{
103+
indexOfOutputFilterToConnect = inputs.size();
104+
indexOfInputPadOfDestinationFilter = index;
105+
}
106+
107+
LOG_INFO("Connect filter " << _filters.at(index)->getName() << " to filter " << _filters.at(indexOfOutputFilterToConnect)->getName())
94108
const int err =
95-
avfilter_link(_filters.at(index)->getAVFilterContext(), 0, _filters.at(index + 1)->getAVFilterContext(), 0);
109+
avfilter_link(_filters.at(index)->getAVFilterContext(), 0, _filters.at(indexOfOutputFilterToConnect)->getAVFilterContext(), indexOfInputPadOfDestinationFilter);
96110
if(err < 0)
97111
{
98112
throw std::runtime_error("Error when connecting filters.");
@@ -128,57 +142,58 @@ void FilterGraph::pushFilter(Filter& filter)
128142
}
129143
}
130144

131-
void FilterGraph::pushInBuffer(const Frame& frame)
145+
void FilterGraph::addInBuffer(const std::vector<Frame*>& inputs)
132146
{
133-
std::string filterName;
134-
std::stringstream filterOptions;
135-
// audio frame
136-
if(frame.isAudioFrame())
147+
for(std::vector<Frame*>::const_reverse_iterator it = inputs.rbegin(); it != inputs.rend(); ++it)
137148
{
138-
filterName = "abuffer";
139-
const AudioFrame& audioFrame = dynamic_cast<const AudioFrame&>(frame);
140-
filterOptions << "time_base=" << _codec.getAVCodecContext().time_base.num << "/"
141-
<< _codec.getAVCodecContext().time_base.den << ":";
142-
filterOptions << "sample_rate=" << audioFrame.getSampleRate() << ":";
143-
filterOptions << "sample_fmt=" << getSampleFormatName(audioFrame.getSampleFormat()) << ":";
144-
filterOptions << "channel_layout=0x" << std::hex << audioFrame.getChannelLayout();
145-
}
146-
// video frame
147-
else if(frame.isVideoFrame())
148-
{
149-
filterName = "buffer";
150-
const VideoFrame& videoFrame = dynamic_cast<const VideoFrame&>(frame);
151-
filterOptions << "video_size=" << videoFrame.getWidth() << "x" << videoFrame.getHeight() << ":";
152-
filterOptions << "pix_fmt=" << getPixelFormatName(videoFrame.getPixelFormat()) << ":";
153-
filterOptions << "time_base=" << _codec.getAVCodecContext().time_base.num << "/"
154-
<< _codec.getAVCodecContext().time_base.den << ":";
155-
filterOptions << "pixel_aspect=" << _codec.getAVCodecContext().sample_aspect_ratio.num << "/"
156-
<< _codec.getAVCodecContext().sample_aspect_ratio.den;
149+
std::string filterName;
150+
std::stringstream filterOptions;
151+
// audio frame
152+
if((*it)->isAudioFrame())
153+
{
154+
filterName = "abuffer";
155+
const AudioFrame* audioFrame = dynamic_cast<const AudioFrame*>(*it);
156+
filterOptions << "time_base=" << _codec.getAVCodecContext().time_base.num << "/"
157+
<< _codec.getAVCodecContext().time_base.den << ":";
158+
filterOptions << "sample_rate=" << audioFrame->getSampleRate() << ":";
159+
filterOptions << "sample_fmt=" << getSampleFormatName(audioFrame->getSampleFormat()) << ":";
160+
filterOptions << "channel_layout=0x" << std::hex << audioFrame->getChannelLayout();
161+
}
162+
// video frame
163+
else if((*it)->isVideoFrame())
164+
{
165+
filterName = "buffer";
166+
const VideoFrame* videoFrame = dynamic_cast<const VideoFrame*>(*it);
167+
filterOptions << "video_size=" << videoFrame->getWidth() << "x" << videoFrame->getHeight() << ":";
168+
filterOptions << "pix_fmt=" << getPixelFormatName(videoFrame->getPixelFormat()) << ":";
169+
filterOptions << "time_base=" << _codec.getAVCodecContext().time_base.num << "/"
170+
<< _codec.getAVCodecContext().time_base.den << ":";
171+
filterOptions << "pixel_aspect=" << _codec.getAVCodecContext().sample_aspect_ratio.num << "/"
172+
<< _codec.getAVCodecContext().sample_aspect_ratio.den;
173+
}
174+
// invalid frame
175+
else
176+
throw std::runtime_error("Cannot create input buffer of filter graph: the given frame is invalid.");
177+
178+
// add in buffer
179+
Filter* in = new Filter(filterName, filterOptions.str(), "in");
180+
LOG_INFO("Add filter '" << filterName << "' at the beginning of the graph.")
181+
_filters.insert(_filters.begin(), in);
157182
}
158-
// invalid frame
159-
else
160-
throw std::runtime_error("Cannot create input buffer of filter graph: the given frame is invalid.");
161-
162-
// add in buffer
163-
Filter* in = new Filter(filterName, filterOptions.str(), "in");
164-
LOG_INFO("Add filter '" << filterName << "' at the beginning of the graph.")
165-
_filters.insert(_filters.begin(), in);
166-
pushFilter(*in);
167183
}
168184

169-
void FilterGraph::pushOutBuffer(const Frame& frame)
185+
void FilterGraph::addOutBuffer(const Frame& output)
170186
{
171187
std::string filterName;
172188

173-
if(frame.isAudioFrame())
189+
if(output.isAudioFrame())
174190
filterName = "abuffersink";
175-
else if(frame.isVideoFrame())
191+
else if(output.isVideoFrame())
176192
filterName = "buffersink";
177193
else
178194
throw std::runtime_error("Cannot create output buffer of filter graph: the given frame is invalid.");
179195

180196
// add out buffer
181-
Filter& out = addFilter(filterName, "", "out");
182-
pushFilter(out);
197+
addFilter(filterName, "", "out");
183198
}
184199
}

0 commit comments

Comments
 (0)