From af42df748cb63b52d7de06f539b4302e480d2292 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Thu, 24 Mar 2016 18:25:02 +0100 Subject: [PATCH 01/11] Frame: renamed copyData to refData --- src/AvTranscoder/data/decoded/Frame.cpp | 2 +- src/AvTranscoder/data/decoded/Frame.hpp | 8 +++++--- src/AvTranscoder/decoder/AudioGenerator.cpp | 4 ++-- src/AvTranscoder/decoder/VideoGenerator.cpp | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/AvTranscoder/data/decoded/Frame.cpp b/src/AvTranscoder/data/decoded/Frame.cpp index 68491b24..612a871e 100644 --- a/src/AvTranscoder/data/decoded/Frame.cpp +++ b/src/AvTranscoder/data/decoded/Frame.cpp @@ -40,7 +40,7 @@ Frame::~Frame() } } -void Frame::copyData(const Frame& frameToRef) +void Frame::refData(const Frame& frameToRef) { const int ret = av_frame_copy(_frame, &frameToRef.getAVFrame()); if(ret < 0) diff --git a/src/AvTranscoder/data/decoded/Frame.hpp b/src/AvTranscoder/data/decoded/Frame.hpp index 8d014891..64bea52c 100644 --- a/src/AvTranscoder/data/decoded/Frame.hpp +++ b/src/AvTranscoder/data/decoded/Frame.hpp @@ -43,9 +43,11 @@ class AvExport Frame int* getLineSize() const { return _frame->linesize; } /** - * @brief Copy the data of the given Frame. - */ - void copyData(const Frame& frameToRef); + * @brief Ref to data of the given Frame. + * @note This function does not allocate anything. + * The current Frame must be already initialized and allocated with the same parameters as frameToRef. + */ + void refData(const Frame& frameToRef); /** * @brief Copy all the fields that do not affect the data layout in the buffers. diff --git a/src/AvTranscoder/decoder/AudioGenerator.cpp b/src/AvTranscoder/decoder/AudioGenerator.cpp index 33309f10..4395dff0 100644 --- a/src/AvTranscoder/decoder/AudioGenerator.cpp +++ b/src/AvTranscoder/decoder/AudioGenerator.cpp @@ -44,12 +44,12 @@ bool AudioGenerator::decodeNextFrame(Frame& frameBuffer) _silent = new AudioFrame(audioBuffer.desc()); } frameBuffer.getAVFrame().nb_samples = _silent->getAVFrame().nb_samples; - frameBuffer.copyData(*_silent); + frameBuffer.refData(*_silent); } // Take audio frame from _inputFrame else { - frameBuffer.copyData(*_inputFrame); + frameBuffer.refData(*_inputFrame); } return true; } diff --git a/src/AvTranscoder/decoder/VideoGenerator.cpp b/src/AvTranscoder/decoder/VideoGenerator.cpp index 74f16a6f..b9c1e9ee 100644 --- a/src/AvTranscoder/decoder/VideoGenerator.cpp +++ b/src/AvTranscoder/decoder/VideoGenerator.cpp @@ -68,12 +68,12 @@ bool VideoGenerator::decodeNextFrame(Frame& frameBuffer) VideoTransform videoTransform; videoTransform.convert(intermediateBuffer, *_blackImage); } - frameBuffer.copyData(*_blackImage); + frameBuffer.refData(*_blackImage); } // Take image from _inputFrame else { - frameBuffer.copyData(*_inputFrame); + frameBuffer.refData(*_inputFrame); } return true; } From d91213eea0c700fbc14e00623c07fe4be1630f4f Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Tue, 29 Mar 2016 18:04:48 +0200 Subject: [PATCH 02/11] IReader: added behavior to generate data when there is no more data to decode * New attribute: _continueWithGenerator * New method: continueWithGenerator --- src/AvTranscoder/reader/IReader.cpp | 17 +++++++++++++++-- src/AvTranscoder/reader/IReader.hpp | 8 ++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/AvTranscoder/reader/IReader.cpp b/src/AvTranscoder/reader/IReader.cpp index 8f7e7630..fe69a8a7 100644 --- a/src/AvTranscoder/reader/IReader.cpp +++ b/src/AvTranscoder/reader/IReader.cpp @@ -16,6 +16,7 @@ IReader::IReader(const std::string& filename, const size_t streamIndex, const in , _channelIndex(channelIndex) , _currentFrame(-1) , _inputFileAllocated(true) + , _continueWithGenerator(false) { _inputFile = new InputFile(filename); } @@ -31,6 +32,7 @@ IReader::IReader(InputFile& inputFile, const size_t streamIndex, const int chann , _channelIndex(channelIndex) , _currentFrame(-1) , _inputFileAllocated(false) + , _continueWithGenerator(false) { } @@ -53,6 +55,7 @@ Frame* IReader::readPrevFrame() Frame* IReader::readFrameAt(const size_t frame) { assert(_decoder != NULL); + assert(_generator != NULL); assert(_transform != NULL); assert(_srcFrame != NULL); assert(_dstFrame != NULL); @@ -70,10 +73,20 @@ Frame* IReader::readFrameAt(const size_t frame) decodingStatus = _decoder->decodeNextFrame(*_srcFrame, _channelIndex); else decodingStatus = _decoder->decodeNextFrame(*_srcFrame); + // if decoding failed if(!decodingStatus) { - _dstFrame->clear(); - return _dstFrame; + // return an empty frame + if(!_continueWithGenerator) + { + _dstFrame->clear(); + return _dstFrame; + } + // or generate data (ie silence or black) + else + { + _generator->decodeNextFrame(*_srcFrame); + } } // transform _transform->convert(*_srcFrame, *_dstFrame); diff --git a/src/AvTranscoder/reader/IReader.hpp b/src/AvTranscoder/reader/IReader.hpp index 51e41083..975c22dc 100644 --- a/src/AvTranscoder/reader/IReader.hpp +++ b/src/AvTranscoder/reader/IReader.hpp @@ -53,10 +53,17 @@ class AvExport IReader */ const StreamProperties* getSourceProperties() const { return _streamProperties; } + /** + * @brief Set the reader state to generate data (ie silence or black) when there is no more data to decode. + * @note By default, the reader returns an empty frame. + */ + void continueWithGenerator(const bool continueWithGenerator = true) { _continueWithGenerator = continueWithGenerator; } + protected: InputFile* _inputFile; const StreamProperties* _streamProperties; IDecoder* _decoder; + IDecoder* _generator; Frame* _srcFrame; Frame* _dstFrame; @@ -69,6 +76,7 @@ class AvExport IReader private: int _currentFrame; ///< The current decoded frame. bool _inputFileAllocated; ///< Does the InputFile is held by the class or not (depends on the constructor called) + bool _continueWithGenerator; ///< If there is no more data to decode, complete with generated data }; } From ed0d9782dd8c2af1ac61a411ce1a46b55bd82c67 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Tue, 29 Mar 2016 18:06:21 +0200 Subject: [PATCH 03/11] Video/AudioReader: added instantiation of a generator To be able to generate data (ie silence or black) when there is no more data to decode. --- src/AvTranscoder/reader/AudioReader.cpp | 5 +++++ src/AvTranscoder/reader/VideoReader.cpp | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/AvTranscoder/reader/AudioReader.cpp b/src/AvTranscoder/reader/AudioReader.cpp index f47bf919..9495d8b5 100644 --- a/src/AvTranscoder/reader/AudioReader.cpp +++ b/src/AvTranscoder/reader/AudioReader.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -42,6 +43,9 @@ void AudioReader::init() _decoder = new AudioDecoder(_inputFile->getStream(_streamIndex)); _decoder->setupDecoder(); + // generator + _generator = new AudioGenerator(); + // create transform _transform = new AudioTransform(); @@ -57,6 +61,7 @@ void AudioReader::init() AudioReader::~AudioReader() { delete _decoder; + delete _generator; delete _srcFrame; delete _dstFrame; delete _transform; diff --git a/src/AvTranscoder/reader/VideoReader.cpp b/src/AvTranscoder/reader/VideoReader.cpp index d5ddd5f7..53fc00f1 100644 --- a/src/AvTranscoder/reader/VideoReader.cpp +++ b/src/AvTranscoder/reader/VideoReader.cpp @@ -1,6 +1,7 @@ #include "VideoReader.hpp" #include +#include #include #include #include @@ -41,6 +42,11 @@ void VideoReader::init() _decoder = new VideoDecoder(_inputFile->getStream(_streamIndex)); _decoder->setupDecoder(); + // generator + VideoGenerator* generator = new VideoGenerator(); + generator->setVideoFrameDesc(_inputFile->getStream(_streamIndex).getVideoCodec().getVideoFrameDesc()); + _generator = generator; + // create transform _transform = new VideoTransform(); @@ -56,6 +62,7 @@ void VideoReader::init() VideoReader::~VideoReader() { delete _decoder; + delete _generator; delete _srcFrame; delete _dstFrame; delete _transform; From 67a147732a26ffa0ee4d567d376eca31ace63bf6 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:38:18 +0200 Subject: [PATCH 04/11] VideoGenerator: added log --- src/AvTranscoder/decoder/VideoGenerator.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/AvTranscoder/decoder/VideoGenerator.cpp b/src/AvTranscoder/decoder/VideoGenerator.cpp index b9c1e9ee..01ac5e50 100644 --- a/src/AvTranscoder/decoder/VideoGenerator.cpp +++ b/src/AvTranscoder/decoder/VideoGenerator.cpp @@ -3,6 +3,8 @@ #include #include +#include + namespace avtranscoder { @@ -51,6 +53,13 @@ bool VideoGenerator::decodeNextFrame(Frame& frameBuffer) // Generate the black image only once if(!_blackImage) { + std::stringstream msg; + msg << "Generate a black image with the following features:" << std::endl; + msg << "width = " << _frameDesc._width << std::endl; + msg << "height = " << _frameDesc._height << std::endl; + msg << "pixel format = rgb24" << std::endl; + LOG_INFO(msg.str()) + VideoFrame& imageBuffer = static_cast(frameBuffer); // Input of convert @@ -68,11 +77,13 @@ bool VideoGenerator::decodeNextFrame(Frame& frameBuffer) VideoTransform videoTransform; videoTransform.convert(intermediateBuffer, *_blackImage); } + LOG_INFO("Reference the black image when decode next frame") frameBuffer.refData(*_blackImage); } // Take image from _inputFrame else { + LOG_INFO("Reference the image data specified when decode next frame") frameBuffer.refData(*_inputFrame); } return true; From 169ab336a0b0fec0250813675ef653062b5de143 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:38:48 +0200 Subject: [PATCH 05/11] AudioGenerator: added log --- src/AvTranscoder/decoder/AudioGenerator.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/AvTranscoder/decoder/AudioGenerator.cpp b/src/AvTranscoder/decoder/AudioGenerator.cpp index 4395dff0..ae93d6e7 100644 --- a/src/AvTranscoder/decoder/AudioGenerator.cpp +++ b/src/AvTranscoder/decoder/AudioGenerator.cpp @@ -1,5 +1,9 @@ #include "AudioGenerator.hpp" +#include + +#include + namespace avtranscoder { @@ -41,14 +45,24 @@ bool AudioGenerator::decodeNextFrame(Frame& frameBuffer) if(!_silent) { AudioFrame& audioBuffer = static_cast(frameBuffer); + + std::stringstream msg; + msg << "Generate a silence with the following features:" << std::endl; + msg << "sample rate = " << audioBuffer.desc()._sampleRate << std::endl; + msg << "number of channels = " << audioBuffer.desc()._nbChannels << std::endl; + msg << "sample format = " << getSampleFormatName(audioBuffer.desc()._sampleFormat) << std::endl; + LOG_INFO(msg.str()) + _silent = new AudioFrame(audioBuffer.desc()); } + LOG_INFO("Reference the silence when decode next frame") frameBuffer.getAVFrame().nb_samples = _silent->getAVFrame().nb_samples; frameBuffer.refData(*_silent); } // Take audio frame from _inputFrame else { + LOG_INFO("Reference the audio data specified when decode next frame") frameBuffer.refData(*_inputFrame); } return true; From 46e9fd68cafc08f95636adcb52884cd0745693dc Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:43:36 +0200 Subject: [PATCH 06/11] IReader: added attribute to avoid updates of dstFrame when return an empty frame * New attribute: _emptyFrame * This commit avoids issue when dstFrame was clear and we need to generate data after this instruction. The dstFrame must not be updated in this case! --- src/AvTranscoder/reader/IReader.cpp | 3 +-- src/AvTranscoder/reader/IReader.hpp | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AvTranscoder/reader/IReader.cpp b/src/AvTranscoder/reader/IReader.cpp index fe69a8a7..6fb9a4f5 100644 --- a/src/AvTranscoder/reader/IReader.cpp +++ b/src/AvTranscoder/reader/IReader.cpp @@ -79,8 +79,7 @@ Frame* IReader::readFrameAt(const size_t frame) // return an empty frame if(!_continueWithGenerator) { - _dstFrame->clear(); - return _dstFrame; + return &_emptyFrame; } // or generate data (ie silence or black) else diff --git a/src/AvTranscoder/reader/IReader.hpp b/src/AvTranscoder/reader/IReader.hpp index 975c22dc..1d55588e 100644 --- a/src/AvTranscoder/reader/IReader.hpp +++ b/src/AvTranscoder/reader/IReader.hpp @@ -77,6 +77,7 @@ class AvExport IReader int _currentFrame; ///< The current decoded frame. bool _inputFileAllocated; ///< Does the InputFile is held by the class or not (depends on the constructor called) bool _continueWithGenerator; ///< If there is no more data to decode, complete with generated data + Frame _emptyFrame; ///< If there is no more data to decode, return an empty frame }; } From e4436cfe1d539179abfb32968621821b625d6815 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:45:03 +0200 Subject: [PATCH 07/11] Frame: removed clear method * Unused. * Warning: after this instruction, we have to set again all fields of the Frame to use it. It is easier to create a new Frame. --- src/AvTranscoder/data/decoded/Frame.cpp | 5 ----- src/AvTranscoder/data/decoded/Frame.hpp | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/AvTranscoder/data/decoded/Frame.cpp b/src/AvTranscoder/data/decoded/Frame.cpp index 612a871e..31b40d54 100644 --- a/src/AvTranscoder/data/decoded/Frame.cpp +++ b/src/AvTranscoder/data/decoded/Frame.cpp @@ -63,11 +63,6 @@ void Frame::refFrame(const Frame& otherFrame) } } -void Frame::clear() -{ - av_frame_unref(_frame); -} - void Frame::allocateAVFrame() { #if LIBAVCODEC_VERSION_MAJOR > 54 diff --git a/src/AvTranscoder/data/decoded/Frame.hpp b/src/AvTranscoder/data/decoded/Frame.hpp index 64bea52c..efc5a20b 100644 --- a/src/AvTranscoder/data/decoded/Frame.hpp +++ b/src/AvTranscoder/data/decoded/Frame.hpp @@ -60,11 +60,6 @@ class AvExport Frame */ void refFrame(const Frame& otherFrame); - /** - * @brief Unreference all the buffers referenced by frame and reset the frame fields. - */ - void clear(); - /** * @return If it corresponds to a valid audio frame. * @see AudioFrame From 8d51ca3ef5c59d3a4b2f3ba2811a80a7010766fc Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:46:06 +0200 Subject: [PATCH 08/11] pyTest: updated testReader to be able to check video frame > 8bits --- test/pyTest/testReader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/pyTest/testReader.py b/test/pyTest/testReader.py index f78a64f8..57b1ee8a 100644 --- a/test/pyTest/testReader.py +++ b/test/pyTest/testReader.py @@ -24,7 +24,8 @@ def testVideoReaderCreateNewInputFile(): # read all frames and check their size for i in xrange(0, reader.getSourceVideoProperties().getNbFrames()): frame = av.VideoFrame(reader.readNextFrame()) - assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * reader.getOutputNbComponents() ) + bytesPerPixel = reader.getOutputBitDepth() / 8 + assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * bytesPerPixel ) # check if the next frame is empty frame = av.VideoFrame(reader.readNextFrame()) @@ -43,7 +44,8 @@ def testVideoReaderReferenceInputFile(): # read all frames and check their size for i in xrange(0, reader.getSourceVideoProperties().getNbFrames()): frame = av.VideoFrame(reader.readNextFrame()) - assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * reader.getOutputNbComponents() ) + bytesPerPixel = reader.getOutputBitDepth() / 8 + assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * bytesPerPixel ) # check if the next frame is empty frame = av.VideoFrame(reader.readNextFrame()) From bdcfc4b198612454c2bcb28f71e5e97bd24bbaa7 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:49:24 +0200 Subject: [PATCH 09/11] pyTest: added tests to check Video/Audio readers with a generator --- test/pyTest/testReader.py | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/pyTest/testReader.py b/test/pyTest/testReader.py index 57b1ee8a..4187bbeb 100644 --- a/test/pyTest/testReader.py +++ b/test/pyTest/testReader.py @@ -76,3 +76,60 @@ def testAudioReaderChannelsExtraction(): sizeOfFrameWithOneChannels = frame.getSize() assert_equals( sizeOfFrameWithAllChannels / nbChannels, sizeOfFrameWithOneChannels ) + + +def testVideoReaderWithGenerator(): + """ + Read a video stream with the VideoReader. + When there is no more data to decode, switch to a generator and process some frames. + """ + inputFileName = os.environ['AVTRANSCODER_TEST_VIDEO_AVI_FILE'] + reader = av.VideoReader(inputFileName) + + # read all frames and check their size + for i in xrange(0, reader.getSourceVideoProperties().getNbFrames()): + frame = av.VideoFrame(reader.readNextFrame()) + bytesPerPixel = reader.getOutputBitDepth() / 8 + assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * bytesPerPixel ) + + # check if the next frame is empty + frame = av.VideoFrame(reader.readNextFrame()) + assert_equals( frame.getSize(), 0 ) + + # generate 10 frames of black + reader.continueWithGenerator() + for i in xrange(0, 9): + frame = av.VideoFrame(reader.readNextFrame()) + bytesPerPixel = reader.getOutputBitDepth() / 8 + assert_equals( frame.getSize(), reader.getOutputWidth() * reader.getOutputHeight() * bytesPerPixel ) + + +def testAudioReaderWithGenerator(): + """ + Read an audio stream with the AudioReader. + When there is no more data to decode, switch to a generator and process some frames. + """ + inputFileName = os.environ['AVTRANSCODER_TEST_AUDIO_WAVE_FILE'] + inputFile = av.InputFile(inputFileName) + reader = av.AudioReader(inputFile) + + # read all frames and check their size + while True: + frame = av.AudioFrame(reader.readNextFrame()) + if frame.getSize() == 0: + break + nbSamplesPerChannel = frame.getNbSamplesPerChannel() + bytesPerSample = 2 + assert_equals( frame.getSize(), reader.getOutputNbChannels() * nbSamplesPerChannel * bytesPerSample ) + + # check if the next frame is empty + frame = av.AudioFrame(reader.readNextFrame()) + assert_equals( frame.getSize(), 0 ) + + # generate 10 frames of silence + reader.continueWithGenerator() + for i in xrange(0, 9): + frame = av.AudioFrame(reader.readNextFrame()) + nbSamplesPerChannel = frame.getNbSamplesPerChannel() + bytesPerSample = 2 + assert_equals( frame.getSize(), reader.getOutputNbChannels() * nbSamplesPerChannel * bytesPerSample ) From 5c31e92d0eb3c3937a10520994eafb8bd9ab1f3a Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 12:49:54 +0200 Subject: [PATCH 10/11] AudioGenerator: fixed assign of audio buffer of silence --- src/AvTranscoder/decoder/AudioGenerator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AvTranscoder/decoder/AudioGenerator.cpp b/src/AvTranscoder/decoder/AudioGenerator.cpp index ae93d6e7..03a57f14 100644 --- a/src/AvTranscoder/decoder/AudioGenerator.cpp +++ b/src/AvTranscoder/decoder/AudioGenerator.cpp @@ -57,7 +57,7 @@ bool AudioGenerator::decodeNextFrame(Frame& frameBuffer) } LOG_INFO("Reference the silence when decode next frame") frameBuffer.getAVFrame().nb_samples = _silent->getAVFrame().nb_samples; - frameBuffer.refData(*_silent); + dynamic_cast(frameBuffer).assign(_silent->getData()[0]); } // Take audio frame from _inputFrame else From 7dd618a4a23d9fa05157472b9a0cc7856af7aec5 Mon Sep 17 00:00:00 2001 From: Clement Champetier Date: Wed, 30 Mar 2016 18:49:25 +0200 Subject: [PATCH 11/11] IReader: refactore instructions if decoding failed Avoid 'if not'/'else': 'if'/'else' is easier. --- src/AvTranscoder/reader/IReader.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/AvTranscoder/reader/IReader.cpp b/src/AvTranscoder/reader/IReader.cpp index 6fb9a4f5..c4eb9b12 100644 --- a/src/AvTranscoder/reader/IReader.cpp +++ b/src/AvTranscoder/reader/IReader.cpp @@ -76,15 +76,15 @@ Frame* IReader::readFrameAt(const size_t frame) // if decoding failed if(!decodingStatus) { - // return an empty frame - if(!_continueWithGenerator) + // generate data (ie silence or black) + if(_continueWithGenerator) { - return &_emptyFrame; + _generator->decodeNextFrame(*_srcFrame); } - // or generate data (ie silence or black) + // or return an empty frame else { - _generator->decodeNextFrame(*_srcFrame); + return &_emptyFrame; } } // transform