Convert camera data to rgb24 with ffmpeg

.gitignore

@@ -0,0 +1,2 @@
+compile_commands.json
+tags

meson.build

@@ -4,13 +4,22 @@ project('magicstore', 'cpp',
 )
 
 libopencv_dep = dependency('opencv4')
+libavcodec_dep = dependency('libavcodec', version: '>=58.134.100')
+libavutil_dep = dependency('libavutil', version: '>=56.70.100')
+libswscale_dep = dependency('libswscale', version: '>=5.9.100')
 
 executable(meson.project_name(),
   'src/main.cpp',
   'src/camera.cpp',
   'src/file_handle.cpp',
+  'src/ffmpeg/avcodec.cpp',
+  'src/ffmpeg/avdeleter.cpp',
+  'src/ffmpeg/decoder.cpp',
   dependencies: [
 	libopencv_dep,
+	libavcodec_dep,
+	libavutil_dep,
+	libswscale_dep,
   ],
   install: true,
 )

src/camera.cpp

@@ -97,8 +97,9 @@ void Camera::start_streaming() {
 	allocate_buffers();
 }
 
-std::vector<char>& Camera::read_frame (std::vector<char>& out_buf) {
+std::vector<std::uint8_t>& Camera::read_frame (std::vector<std::uint8_t>& out_buf) {
-	const char* const buffer = m_local->input_buffer;
+	static_assert(sizeof(char) == sizeof(std::uint8_t), "Unsupported architecture");
+	const std::uint8_t* const buffer = reinterpret_cast<const std::uint8_t*>(m_local->input_buffer);
 	v4l2_buffer& buffer_info = m_local->buffer_info;
 
 	// Queue the buffer

src/camera.hpp

@@ -34,7 +34,7 @@ public:
 	FileHandle file_handle() const;
 
 	void start_streaming();
-	std::vector<char>& read_frame (std::vector<char>& out_buf);
+	std::vector<std::uint8_t>& read_frame (std::vector<std::uint8_t>& out_buf);
 	FrameFormat frame_size() const;
 	void set_frame_size (const FrameFormat& new_size);
 	unsigned int stride() const;

src/ffmpeg/avcodec.cpp

@@ -0,0 +1,51 @@
+#include "avcodec.hpp"
+extern "C" {
+#include <libavcodec/avcodec.h>
+} //extern C
+#include <stdexcept>
+#include <string>
+
+namespace mgs::ffmpeg {
+namespace {
+::AVPixelFormat get_favourite_format_ifp (::AVCodecContext* s, const ::AVPixelFormat* fmt) {
+	const ::AVPixelFormat* curr_fmt = fmt;
+	const auto fav_format = static_cast<::AVPixelFormat>(Decoder::FavouriteFormat);
+	while (*curr_fmt != -1) {
+		if (*curr_fmt == fav_format)
+			return fav_format;
+		++curr_fmt;
+	}
+	return ::avcodec_default_get_format(s, fmt);
+}
+} //unnamed namespace
+
+AVCodec::AVCodec (::AVCodecID id) :
+	m_avcodec(::avcodec_find_decoder(id))
+{
+	if (not m_avcodec)
+		throw std::runtime_error("Unable to finde avcodec with id " + std::to_string(id));
+}
+
+AVCodec::~AVCodec() noexcept = default;
+
+Decoder AVCodec::make_decoder (unsigned int width, unsigned int height) {
+	::AVCodec* const codec = m_avcodec.get();
+	Decoder::UniqueAVCodecContext context{::avcodec_alloc_context3(codec)};
+	if (not context)
+		throw std::runtime_error("Could not allocate video codec context");
+
+
+	if (codec->capabilities & AV_CODEC_CAP_TRUNCATED)
+		context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames
+
+	context->width = width;
+	context->height = height;
+	context->get_format = &get_favourite_format_ifp;
+
+	if (::avcodec_open2(context.get(), codec, nullptr) < 0)
+		throw std::runtime_error("Could not open codec");
+
+	return {std::move(context)};
+}
+
+} //namespace mgs::ffmpeg

src/ffmpeg/avcodec.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "decoder.hpp"
+#include "avdeleter.hpp"
+extern "C" {
+#include <libavcodec/codec_id.h>
+} //extern C
+
+namespace mgs::ffmpeg {
+
+class AVCodec {
+public:
+	AVCodec() = delete;
+	explicit AVCodec (::AVCodecID id);
+	~AVCodec() noexcept;
+
+	Decoder make_decoder (unsigned int width, unsigned int height);
+
+private:
+	unique_av_ptr<struct ::AVCodec> m_avcodec;
+};
+} //namespace mgs::ffmpeg

src/ffmpeg/avdeleter.cpp

@@ -0,0 +1,66 @@
+#include "avdeleter.hpp"
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+} //extern C
+#if !defined(NDEBUG)
+#	include <iostream>
+#endif
+
+namespace mgs::ffmpeg {
+namespace {
+void delete_avcodec_object (::AVCodec*) noexcept {
+	//no close function for this one
+#if !defined(NDEBUG)
+	std::cout << "AVCodec object freed\n";
+#endif
+}
+
+void delete_avcodec_object (::AVCodecContext* ptr) noexcept {
+	//avcodec_close(ptr);
+	//av_free(ptr);
+	avcodec_free_context(&ptr);
+#if !defined(NDEBUG)
+	std::cout << "AVCodecContext object closed and freed\n";
+#endif
+}
+
+void delete_avcodec_object (::AVFrame* ptr) noexcept {
+	av_frame_free(&ptr);
+#if !defined(NDEBUG)
+	std::cout << "AVFrame object freed\n";
+#endif
+}
+
+void delete_avcodec_object (::AVPacket* ptr) noexcept {
+	av_packet_free(&ptr);
+#if !defined(NDEBUG)
+	std::cout << "AVPacket object freed\n";
+#endif
+}
+
+void delete_avcodec_object (::SwsContext* ptr) noexcept {
+	sws_freeContext(ptr);
+#if !defined(NDEBUG)
+	std::cout << "SwsContext object freed\n";
+#endif
+}
+
+void delete_avcodec_object (std::uint8_t* ptr) noexcept {
+	::av_free(ptr);
+}
+} //unnamed namespace
+
+template <typename T>
+void AVDeleter<T>::operator() (T* ptr) const noexcept {
+	delete_avcodec_object(ptr);
+}
+
+template struct AVDeleter<::AVCodec>;
+template struct AVDeleter<::AVCodecContext>;
+template struct AVDeleter<::AVPacket>;
+template struct AVDeleter<::AVFrame>;
+template struct AVDeleter<::SwsContext>;
+template struct AVDeleter<std::uint8_t>;
+
+} // namespace mgs::ffmpeg

src/ffmpeg/avdeleter.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <memory>
+
+struct AVCodec;
+struct AVCodecContext;
+struct AVPacket;
+struct AVFrame;
+struct SwsContext;
+
+namespace mgs::ffmpeg {
+template <typename T>
+struct AVDeleter {
+	void operator()(T*) const noexcept;
+};
+
+template <typename T>
+using unique_av_ptr = std::unique_ptr< T, AVDeleter<T> >;
+} //namespace mgs::ffmpeg

src/ffmpeg/decoder.cpp

@@ -0,0 +1,112 @@
+#include "decoder.hpp"
+#include <utility>
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+#include <libavutil/imgutils.h>
+} //extern C
+#include <algorithm>
+#include <cassert>
+#include <stdexcept>
+
+namespace mgs::ffmpeg {
+namespace {
+constexpr int Align = 32;
+} //unnamed namespace
+
+const int Decoder::FavouriteFormat = static_cast<int>(::AVPixelFormat::AV_PIX_FMT_RGB24);
+
+Decoder::Decoder (UniqueAVCodecContext&& context) :
+	m_context(std::move(context)),
+	m_packet(::av_packet_alloc()),
+	m_frame1(::av_frame_alloc()),
+	m_frame2(::av_frame_alloc())
+{
+}
+
+Decoder::~Decoder() noexcept = default;
+
+std::size_t Decoder::decode (const std::uint8_t* input, std::size_t input_size, std::uint8_t* out, std::size_t out_size) {
+	//from https://stackoverflow.com/questions/55131040/how-to-decode-mjpeg-with-ffmpeg
+	if (not input_size)
+		return 0;
+
+	UniqueAVMem buf_data {static_cast<std::uint8_t*>(::av_malloc(input_size))};
+	std::copy(input, input + input_size, buf_data.get());
+	if (::av_packet_from_data (m_packet.get(), buf_data.get(), input_size))
+		throw std::runtime_error("Setting packet data failed");
+	buf_data.release();
+
+	const int send_packet_ret = ::avcodec_send_packet(m_context.get(), m_packet.get());
+	::av_packet_unref(m_packet.get());
+	if (send_packet_ret < 0)
+		return 0;
+
+	if (not m_sws_context)
+		init_sws_context();
+
+	const int recv_frame_ret = ::avcodec_receive_frame(m_context.get(), m_frame1.get());
+	if (recv_frame_ret < 0)
+		return 0;
+
+	const std::size_t new_out_size = ::av_image_get_buffer_size(
+		static_cast<::AVPixelFormat>(FavouriteFormat),
+		m_context->width,
+		m_context->height,
+		Align
+	);
+
+	assert(m_frame2->data[0]);
+	::sws_scale(
+		m_sws_context.get(),
+		m_frame1->data,
+		m_frame1->linesize,
+		0,
+		m_frame1->height,
+		m_frame2->data,
+		m_frame2->linesize
+	);
+	const std::size_t out_size_safe = std::min(out_size, new_out_size);
+	std::copy(m_frame2->data[0], m_frame2->data[0] + out_size_safe, out);
+
+	return out_size_safe;
+}
+
+void Decoder::init_sws_context() {
+	const std::size_t new_out_size = ::av_image_get_buffer_size(
+		static_cast<::AVPixelFormat>(FavouriteFormat),
+		m_context->width,
+		m_context->height,
+		Align
+	);
+
+	//are you calling this method before ffmpeg had a chance to autodetect
+	//the correct src pix format?
+	//it should be a side effect of avcodec_send_packet()
+	assert(::AVPixelFormat::AV_PIX_FMT_NONE != m_context->pix_fmt);
+
+	m_sws_context = UniqueSwsContext{::sws_getContext(
+		m_context->width, //int src_width
+		m_context->height, //int src_height
+		m_context->pix_fmt, //AV_PIX_FMT_YUVJ422P, //AVPixelFormat src_format
+		m_context->width, //int dst_width
+		m_context->height, //int dst_height
+		static_cast<::AVPixelFormat>(FavouriteFormat), //AVPixelFormat dst_format
+		SWS_FAST_BILINEAR, //int flags
+		nullptr, //SwsFilter* src_filter
+		nullptr, //SwsFilter* dst_filter
+		nullptr //const double* param
+	)};
+	m_frame2_mem.reset(static_cast<std::uint8_t*>(::av_malloc(new_out_size)));
+	::av_image_fill_arrays(
+		m_frame2->data,
+		m_frame2->linesize,
+		m_frame2_mem.get(),
+		static_cast<::AVPixelFormat>(FavouriteFormat),
+		m_context->width,
+		m_context->height,
+		Align
+	);
+}
+
+} //namespace mgs::ffmpeg

src/ffmpeg/decoder.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "avdeleter.hpp"
+#include <cstddef>
+#include <cstdint>
+
+namespace mgs::ffmpeg {
+class Decoder {
+public:
+	typedef unique_av_ptr<struct ::AVCodecContext> UniqueAVCodecContext;
+	typedef unique_av_ptr<struct ::AVFrame> UniqueAVFrame;
+	typedef unique_av_ptr<struct ::AVPacket> UniqueAVPacket;
+	typedef unique_av_ptr<struct ::SwsContext> UniqueSwsContext;
+	typedef unique_av_ptr<std::uint8_t> UniqueAVMem;
+
+	static const int FavouriteFormat;
+
+	Decoder (UniqueAVCodecContext&& context);
+	~Decoder() noexcept;
+	std::size_t decode (const std::uint8_t* input, std::size_t input_size, std::uint8_t* out, std::size_t out_size);
+
+private:
+	void init_sws_context();
+
+	UniqueAVMem m_frame2_mem;
+	UniqueAVCodecContext m_context;
+	UniqueAVPacket m_packet;
+	UniqueAVFrame m_frame1;
+	UniqueAVFrame m_frame2;
+	UniqueSwsContext m_sws_context;
+};
+} //namespace mgs::ffmpeg

src/main.cpp

@@ -1,4 +1,5 @@
 #include "camera.hpp"
+#include "ffmpeg/avcodec.hpp"
 #if !defined(NDEBUG)
 #	include <iostream>
 #	include <iomanip>
@@ -45,14 +46,28 @@ int main() {
 		mgs::Camera cam("/dev/video0", {}, false);
 		cam.set_frame_size({1280, 720, MjpegFormat});
 
+		mgs::ffmpeg::AVCodec avcodec{::AVCodecID::AV_CODEC_ID_MJPEG};
+		auto cam_frame_format = cam.frame_size();
+		auto decoder{avcodec.make_decoder(cam_frame_format.width, cam_frame_format.height)};
+
 		print_supported_frame_formats(cam);
 		cam.start_streaming();
 
-		std::vector<char> my_frame;
+		std::vector<std::uint8_t> my_frame;
 		auto data = cam.read_frame(my_frame);
+		std::cout << "Read frame of " << data.size() << " bytesfrom camera\n";
 
 		std::ofstream file_wr("temp.jpg", std::ios::binary | std::ios::out);
-		file_wr.write(data.data(), data.size());
+		file_wr.write(reinterpret_cast<const char*>(data.data()), data.size());
+
+		std::vector<std::uint8_t> buff;
+		buff.resize(cam_frame_format.width * 3 * cam_frame_format.height);
+		const auto out_data_size = decoder.decode(data.data(), data.size(), buff.data(), buff.size());
+		std::cout << "Allocated buffer size: " << buff.size() << '\n';
+		std::cout << "Decoded buffer size: " << out_data_size << '\n';
+
+		std::ofstream rgb_file_wr("rgb.data", std::ios::binary | std::ios::out);
+		rgb_file_wr.write(reinterpret_cast<const char*>(buff.data()), buff.size());
 	}
 	catch (const std::runtime_error& err) {
 		std::cerr << "Unhandled exception: " << err.what() << '\n';