How to integrate FFmpeg in applications? What are the common API functions?

FFmpeg not only provides command-line tools but also rich C/C++ APIs, allowing developers to integrate audio/video processing capabilities into their applications.

Core Libraries

FFmpeg mainly includes the following core libraries:

Library	Function	Purpose
libavformat	Container format handling	Read/write various audio/video container formats
libavcodec	Codec	Audio/video encoding and decoding
libavutil	Utility functions	Common utilities and helper functions
libswscale	Image scaling	Image scaling and color space conversion
libswresample	Audio resampling	Audio sample rate conversion and format conversion
libavfilter	Filter processing	Audio/video filter effects
libavdevice	Device support	Camera, microphone and other device access

Basic Development Workflow

Initialize FFmpeg

c
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>

// Register all codecs and formats
av_register_all();
avformat_network_init();

// FFmpeg 4.0+ does not need explicit registration

Open Input File

c
AVFormatContext *fmt_ctx = NULL;
int ret = avformat_open_input(&fmt_ctx, "input.mp4", NULL, NULL);
if (ret < 0) {
    // Handle error
}

// Get stream information
ret = avformat_find_stream_info(fmt_ctx, NULL);
if (ret < 0) {
    // Handle error
}

Find Video Stream

c
int video_stream_index = -1;
for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) {
    if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
        video_stream_index = i;
        break;
    }
}

Initialize Decoder

c
AVCodecParameters *codec_par = fmt_ctx->streams[video_stream_index]->codecpar;
const AVCodec *codec = avcodec_find_decoder(codec_par->codec_id);
AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);

avcodec_parameters_to_context(codec_ctx, codec_par);
ret = avcodec_open2(codec_ctx, codec, NULL);
if (ret < 0) {
    // Handle error
}

Read and Decode Frames

c
AVPacket *packet = av_packet_alloc();
AVFrame *frame = av_frame_alloc();

while (av_read_frame(fmt_ctx, packet) >= 0) {
    if (packet->stream_index == video_stream_index) {
        ret = avcodec_send_packet(codec_ctx, packet);
        if (ret < 0) {
            // Handle error
        }

        while (ret >= 0) {
            ret = avcodec_receive_frame(codec_ctx, frame);
            if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
                break;
            } else if (ret < 0) {
                // Handle error
            }
            
            // Process decoded frame
            process_frame(frame);
        }
    }
    av_packet_unref(packet);
}

Encode Output

c
// Initialize encoder
const AVCodec *encoder = avcodec_find_encoder(AV_CODEC_ID_H264);
AVCodecContext *enc_ctx = avcodec_alloc_context3(encoder);

enc_ctx->width = 1280;
enc_ctx->height = 720;
enc_ctx->time_base = (AVRational){1, 25};
enc_ctx->framerate = (AVRational){25, 1};
enc_ctx->gop_size = 10;
enc_ctx->max_b_frames = 1;
enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P;

ret = avcodec_open2(enc_ctx, encoder, NULL);
if (ret < 0) {
    // Handle error
}

// Encode frame
AVPacket *enc_packet = av_packet_alloc();
ret = avcodec_send_frame(enc_ctx, frame);
if (ret < 0) {
    // Handle error
}

while (ret >= 0) {
    ret = avcodec_receive_packet(enc_ctx, enc_packet);
    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
        break;
    } else if (ret < 0) {
        // Handle error
    }
    
    // Write to output file
    av_interleaved_write_frame(out_fmt_ctx, enc_packet);
    av_packet_unref(enc_packet);
}

Resource Cleanup

c
av_frame_free(&frame);
av_packet_free(&packet);
avcodec_free_context(&codec_ctx);
avformat_close_input(&fmt_ctx);

Common API Functions

Format Handling

c
// Open input
int avformat_open_input(AVFormatContext **ps, const char *url, AVInputFormat *fmt, AVDictionary **options);

// Find stream information
int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options);

// Read frame
int av_read_frame(AVFormatContext *s, AVPacket *pkt);

// Write frame
int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt);

Codec

c
// Find decoder
const AVCodec *avcodec_find_decoder(enum AVCodecID id);

// Find encoder
const AVCodec *avcodec_find_encoder(enum AVCodecID id);

// Open codec
int avcodec_open2(AVCodecContext *avctx, const AVCodec *codec, AVDictionary **options);

// Send packet to decoder
int avcodec_send_packet(AVCodecContext *avctx, const AVPacket *avpkt);

// Receive frame from decoder
int avcodec_receive_frame(AVCodecContext *avctx, AVFrame *frame);

// Send frame to encoder
int avcodec_send_frame(AVCodecContext *avctx, const AVFrame *frame);

// Receive packet from encoder
int avcodec_receive_packet(AVCodecContext *avctx, AVPacket *avpkt);

Image Processing

c
// Allocate frame
AVFrame *av_frame_alloc(void);

// Allocate frame data
int av_frame_get_buffer(AVFrame *frame, int align);

// Image scaling
struct SwsContext *sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat,
                                   int dstW, int dstH, enum AVPixelFormat dstFormat,
                                   int flags, SwsFilter *srcFilter,
                                   SwsFilter *dstFilter, const double *param);

int sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[],
              const int srcStride[], int srcSliceY, int srcSliceH,
              uint8_t *const dst[], const int dstStride[]);

Error Handling

c
// Get error description
char errbuf[AV_ERROR_MAX_STRING_SIZE];
av_strerror(ret, errbuf, AV_ERROR_MAX_STRING_SIZE);
fprintf(stderr, "Error: %s\n", errbuf);

// Common error codes
AVERROR(EAGAIN)      // Need more input/output
AVERROR_EOF          // End of file
AVERROR(EINVAL)      // Invalid parameter
AVERROR(ENOMEM)      // Out of memory

Best Practices

1. Resource Management: Use RAII pattern to manage resources and ensure proper cleanup
2. Error Handling: Check return values of all API calls
3. Thread Safety: Most FFmpeg APIs are not thread-safe, need proper synchronization
4. Performance Optimization: Use hardware acceleration, multi-threading
5. Memory Management: Release resources promptly when no longer needed

FFmpeg API is powerful but complex. It is recommended to start with simple examples and gradually master the usage of each module.