lib/frameFilter.cpp

/*
 *  Copyright (C) 2020-2021 Savoir-faire Linux Inc.
 *
 *  Author: Aline Gondim Santos <aline.gondimsantos@savoirfairelinux.com>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA.
 */

#include "frameFilter.h"

extern "C" {
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
}

#include <algorithm>
#include <functional>
#include <memory>
#include <sstream>
#include <thread>

#include "pluglog.h"

namespace jami {

FrameFilter::FrameFilter() {}

FrameFilter::~FrameFilter()
{
    clean();
}

int
FrameFilter::initialize(const std::string& filterDesc, std::vector<MediaStream> msps)
{
    int ret = 0;
    desc_ = filterDesc;
    graph_ = avfilter_graph_alloc();

    if (!graph_)
        return fail("Failed to allocate filter graph", AVERROR(ENOMEM));

    graph_->nb_threads = std::max(1u, std::min(8u, std::thread::hardware_concurrency() / 2));

    AVFilterInOut* in;
    AVFilterInOut* out;
    if ((ret = avfilter_graph_parse2(graph_, desc_.c_str(), &in, &out)) < 0)
        return fail("Failed to parse filter graph", ret);

    using AVFilterInOutPtr = std::unique_ptr<AVFilterInOut, std::function<void(AVFilterInOut*)>>;
    AVFilterInOutPtr outputs(out, [](AVFilterInOut* f) { avfilter_inout_free(&f); });
    AVFilterInOutPtr inputs(in, [](AVFilterInOut* f) { avfilter_inout_free(&f); });

    if (outputs && outputs->next)
        return fail("Filters with multiple outputs are not supported", AVERROR(ENOTSUP));

    if ((ret = initOutputFilter(outputs.get())) < 0)
        return fail("Failed to create output for filter graph", ret);
    // make sure inputs linked list is the same size as msps
    size_t count = 0;
    AVFilterInOut* dummyInput = inputs.get();
    while (dummyInput && ++count) // increment count before evaluating its value
        dummyInput = dummyInput->next;
    if (count != msps.size())
        return fail(
            "Size mismatch between number of inputs in filter graph and input parameter array",
            AVERROR(EINVAL));
    for (AVFilterInOut* current = inputs.get(); current; current = current->next) {
        if (!current->name)
            return fail("Filters require non empty names", AVERROR(EINVAL));
        std::string name = current->name;
        const auto& it = std::find_if(msps.begin(), msps.end(), [name](const MediaStream& msp) {
            return msp.name == name;
        });
        if (it != msps.end()) {
            if ((ret = initInputFilter(current, *it)) < 0) {
                std::string msg = "Failed to initialize input: " + name;
                return fail(msg, ret);
            }
        } else {
            std::string msg = "Failed to find matching parameters for: " + name;
            return fail(msg, ret);
        }
    }

    if ((ret = avfilter_graph_config(graph_, nullptr)) < 0)
        return fail("Failed to configure filter graph", ret);

    initialized_ = true;
    return 0;
}

MediaStream
FrameFilter::getInputParams(const std::string& inputName) const
{
    for (auto ms : inputParams_)
        if (ms.name == inputName)
            return ms;
    return {};
}

MediaStream
FrameFilter::getOutputParams() const
{
    MediaStream output;
    if (!output_ || !initialized_) {
        fail("Filter not initialized", -1);
        return output;
    }

    switch (av_buffersink_get_type(output_)) {
    case AVMEDIA_TYPE_VIDEO:
        output.name = "videoOutput";
        output.format = av_buffersink_get_format(output_);
        output.isVideo = true;
        output.timeBase = av_buffersink_get_time_base(output_);
        output.width = av_buffersink_get_w(output_);
        output.height = av_buffersink_get_h(output_);
        output.bitrate = 0;
        output.frameRate = av_buffersink_get_frame_rate(output_);
        break;
    case AVMEDIA_TYPE_AUDIO:
        output.name = "audioOutput";
        output.format = av_buffersink_get_format(output_);
        output.isVideo = false;
        output.timeBase = av_buffersink_get_time_base(output_);
        output.sampleRate = av_buffersink_get_sample_rate(output_);
        output.nbChannels = av_buffersink_get_channels(output_);
        break;
    default:
        output.format = -1;
        break;
    }
    return output;
}

int
FrameFilter::feedEOF(const std::string& inputName)
{
    int ret = 0;
    if (!initialized_)
        return fail("Filter not initialized", -1);

    for (size_t i = 0; i < inputs_.size(); ++i) {
        auto& ms = inputParams_[i];
        if (ms.name != inputName)
            continue;

        if ((ret = av_buffersrc_add_frame_flags(inputs_[i], nullptr, AV_BUFFERSRC_FLAG_PUSH)) < 0)
            return fail("Could not pass frame to filters", ret);
        else
            return 0;
    }

    std::stringstream ss;
    ss << "Specified filter (" << inputName << ") not found";
    return fail(ss.str(), AVERROR(EINVAL));
}

int
FrameFilter::feedInput(AVFrame* frame, const std::string& inputName)
{
    int ret = 0;
    if (!initialized_)
        return fail("Filter not initialized", -1);

    if (!frame)
        return 0;

    for (size_t i = 0; i < inputs_.size(); ++i) {
        auto& ms = inputParams_[i];
        if (ms.name != inputName)
            continue;

        if (ms.format != frame->format
            || (ms.isVideo && (ms.width != frame->width || ms.height != frame->height))
            || (!ms.isVideo
                && (ms.sampleRate != frame->sample_rate || ms.nbChannels != frame->ch_layout.nb_channels))) {
            ms.update(frame);
            if ((ret = reinitialize()) < 0)
                return fail("Failed to reinitialize filter with new input parameters", ret);
        }

        if ((ret = av_buffersrc_add_frame_flags(inputs_[i], frame, AV_BUFFERSRC_FLAG_KEEP_REF)) < 0)
            return fail("Could not pass frame to filters", ret);
        else
            return 0;
    }

    std::stringstream ss;
    ss << "Specified filter (" << inputName << ") not found";
    return fail(ss.str(), AVERROR(EINVAL));
}

AVFrame*
FrameFilter::readOutput()
{
    if (!initialized_) {
        fail("Not properly initialized", -1);
        return nullptr;
    }

    AVFrame* frame = av_frame_alloc();

    auto type = av_buffersink_get_type(output_);
    if (type != AVMEDIA_TYPE_VIDEO && type != AVMEDIA_TYPE_AUDIO) {
        av_frame_unref(frame);
        av_frame_free(&frame);
        return nullptr;
    }
    auto err = av_buffersink_get_frame(output_, frame);
    if (err >= 0) {
        return frame;
    } else if (err == AVERROR(EAGAIN)) {
        fail("no data. ", err);
    } else if (err == AVERROR_EOF) {
        fail("eof. ", err);
    } else {
        fail("Error occurred while pulling from filter graph", err);
    }
    av_frame_unref(frame);
    av_frame_free(&frame);
    return nullptr;
}

int
FrameFilter::initOutputFilter(AVFilterInOut* out)
{
    int ret = 0;
    const AVFilter* buffersink;
    AVFilterContext* buffersinkCtx{};
    AVMediaType mediaType = avfilter_pad_get_type(out->filter_ctx->input_pads, out->pad_idx);

    if (mediaType == AVMEDIA_TYPE_VIDEO)
        buffersink = avfilter_get_by_name("buffersink");
    else
        buffersink = avfilter_get_by_name("abuffersink");

    if ((ret
         = avfilter_graph_create_filter(&buffersinkCtx, buffersink, "out", nullptr, nullptr, graph_))
        < 0) {
        avfilter_free(buffersinkCtx);
        return fail("Failed to create buffer sink", ret);
    }

    if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, buffersinkCtx, 0)) < 0) {
        avfilter_free(buffersinkCtx);
        return fail("Could not link buffer sink to graph", ret);
    }

    output_ = buffersinkCtx;
    return ret;
}

int
FrameFilter::initInputFilter(AVFilterInOut* in, MediaStream msp)
{
    int ret = 0;
    AVBufferSrcParameters* params = av_buffersrc_parameters_alloc();
    if (!params)
        return -1;

    const AVFilter* buffersrc;
    AVMediaType mediaType = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
    params->format = msp.format;
    params->time_base = msp.timeBase;
    if (mediaType == AVMEDIA_TYPE_VIDEO) {
        params->width = msp.width;
        params->height = msp.height;
        params->frame_rate = msp.frameRate;
        buffersrc = avfilter_get_by_name("buffer");
    } else {
        params->sample_rate = msp.sampleRate;
        av_channel_layout_default(&(params->ch_layout), msp.nbChannels);
        buffersrc = avfilter_get_by_name("abuffer");
    }

    AVFilterContext* buffersrcCtx{};
    if (buffersrc) {
        char name[128];
        snprintf(name, sizeof(name), "buffersrc_%s_%d", in->name, in->pad_idx);
        buffersrcCtx = avfilter_graph_alloc_filter(graph_, buffersrc, name);
    }
    if (!buffersrcCtx) {
        av_free(params);
        return fail("Failed to allocate filter graph input", AVERROR(ENOMEM));
    }
    ret = av_buffersrc_parameters_set(buffersrcCtx, params);
    av_free(params);
    if (ret < 0)
        return fail("Failed to set filter graph input parameters", ret);

    if ((ret = avfilter_init_str(buffersrcCtx, nullptr)) < 0)
        return fail("Failed to initialize buffer source", ret);

    if ((ret = avfilter_link(buffersrcCtx, 0, in->filter_ctx, in->pad_idx)) < 0)
        return fail("Failed to link buffer source to graph", ret);

    inputs_.push_back(buffersrcCtx);
    msp.name = in->name;
    inputParams_.push_back(msp);
    return ret;
}

int
FrameFilter::reinitialize()
{
    // keep parameters needed for initialization before clearing filter
    auto params = std::move(inputParams_);
    auto desc = std::move(desc_);
    clean();
    auto ret = initialize(desc, params);
    return ret;
}

int
FrameFilter::fail(std::string msg, int err) const
{
    if (!msg.empty())
        Plog::log(Plog::LogPriority::INFO, "Frame Filter: ", "("+std::to_string(err)+") "+msg);
    return err;
}

void
FrameFilter::clean()
{
    initialized_ = false;
    avfilter_graph_free(&graph_); // frees inputs_ and output_
    desc_.clear();
    inputs_.clear();   // don't point to freed memory
    output_ = nullptr; // don't point to freed memory
    inputParams_.clear();
}
} // namespace jami