foreground: modify post processing
This commit unifies the post processing step for every supported platform.
Change-Id: Iab32808357824cd75f694aa9d0f02ffe0642951c
diff --git a/ForegroundSegmentation/buildandroid.sh b/ForegroundSegmentation/buildandroid.sh
index 8170f18..d9ab3ec 100644
--- a/ForegroundSegmentation/buildandroid.sh
+++ b/ForegroundSegmentation/buildandroid.sh
@@ -9,7 +9,7 @@
echo "ANDROID_NDK not provided, building with ${ANDROID_NDK}"
fi
-PLUGIN_NAME="foregroungsegmentation"
+PLUGIN_NAME="foregroundsegmentation"
JPL_FILE_NAME=${PLUGIN_NAME}".jpl"
SO_FILE_NAME="lib"${PLUGIN_NAME}".so"
LIBS_DIR="/home/${USER}/Libs"
@@ -44,7 +44,7 @@
export LD=$TOOLCHAIN/bin/arm-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/arm-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/arm-linux-androideabi-strip
- export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm/sysroot
+ export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm/sysroot
elif [ $CURRENT_ABI = arm64-v8a ]
then
@@ -55,7 +55,7 @@
export LD=$TOOLCHAIN/bin/aarch64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/aarch64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/aarch64-linux-android-strip
- export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-arm64/sysroot
+ export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-arm64/sysroot
elif [ $CURRENT_ABI = x86_64 ]
then
@@ -66,13 +66,13 @@
export LD=$TOOLCHAIN/bin/x86_64-linux-android-ld
export RANLIB=$TOOLCHAIN/bin/x86_64-linux-android-ranlib
export STRIP=$TOOLCHAIN/bin/x86_64-linux-android-strip
- export ANDROID_SYSROOT=/home/${USER}/Projects/ring-android-project/client-android/android-toolchain-21-x86_64/sysroot
+ export ANDROID_SYSROOT=./../../client-android/android-toolchain-21-x86_64/sysroot
else
echo "ABI NOT OK" >&2
exit 1
fi
-
+
#=========================================================
# CONTRIBS
#=========================================================
@@ -88,14 +88,14 @@
then
CONTRIB_PLATFORM=x86_64-linux-android
fi
-
+
# ASSETS
- ANDROID_PROJECT_ASSETS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/assets
+ ANDROID_PROJECT_ASSETS=./../../client-android/ring-android/app/src/main/assets
# LIBS FOLDER
- ANDROID_PROJECT_LIBS=/home/${USER}/Projects/ring-android-project/client-android/ring-android/app/src/main/libs/$CURRENT_ABI
+ ANDROID_PROJECT_LIBS=./../../client-android/ring-android/app/src/main/libs/$CURRENT_ABI
#NDK SOURCES FOR cpufeatures
NDK_SOURCES=${ANDROID_NDK}/sources/android
-
+
#=========================================================
# LD_FLAGS
#=========================================================
@@ -109,7 +109,7 @@
then
export EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android -L${ANDROID_SYSROOT}/usr/lib/x86_64-linux-android/21"
fi
-
+
#=========================================================
# Compile CPU FEATURES, NEEDED FOR OPENCV
#=========================================================
@@ -118,7 +118,7 @@
#=========================================================
# Compile the plugin
#=========================================================
-
+
# Create so destination folder
mkdir -p lib/$CURRENT_ABI
@@ -160,11 +160,11 @@
-o lib/$CURRENT_ABI/${SO_FILE_NAME}
# (above) Always put opencv_core after all other opencv libs when linking statically
# (above) Put libavutil after other ffmpeg libraries
-
+
cp ${LIBS_DIR}/_tensorflow_distribution/lib/${CURRENT_ABI}/libtensorflowlite.so lib/$CURRENT_ABI
}
-# Build the so
+# Build the so
for i in ${ANDROID_ABI}; do
CURRENT_ABI=$i
buildlib
diff --git a/ForegroundSegmentation/data/preferences.json b/ForegroundSegmentation/data/preferences.json
index 372215b..88d1778 100644
--- a/ForegroundSegmentation/data/preferences.json
+++ b/ForegroundSegmentation/data/preferences.json
@@ -15,7 +15,7 @@
"key": "modellist",
"title": "Model to load",
"summary": "Select the model to use",
- "defaultValue": "model_256_F_16.tflite",
+ "defaultValue": "model_256_Qlatency.tflite",
"entries": ["mv2_DLV3_256_MQ", "mv2_DLV3_256_QLATENCY_16", "mv2_DLV3_256_QLATENCY_8"],
"entryValues": ["mobilenet_v2_deeplab_v3_256_myquant.tflite", "model_256_Qlatency_16.tflite", "model_256_Qlatency.tflite"]
},
diff --git a/ForegroundSegmentation/pluginProcessor.cpp b/ForegroundSegmentation/pluginProcessor.cpp
index 6b215a4..35490aa 100644
--- a/ForegroundSegmentation/pluginProcessor.cpp
+++ b/ForegroundSegmentation/pluginProcessor.cpp
@@ -30,12 +30,10 @@
backgroundImage = cv::imread(backgroundPath);
if (backgroundImage.cols == 0)
{
- Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
+ Plog::log(Plog::LogPriority::ERROR, TAG, "Background image not Loaded");
}
cv::cvtColor(backgroundImage, backgroundImage, cv::COLOR_BGR2RGB);
-#ifndef __ANDROID__
backgroundImage.convertTo(backgroundImage, CV_32FC3);
-#endif
//TODO: properly resize the background image to maintain background aspect ratio in the output image;
Plog::log(Plog::LogPriority::INFO, TAG, mPluginParameters->model);
}
@@ -43,7 +41,7 @@
void PluginProcessor::initModel()
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside getImageNbChannels()");
- try {
+ try {
pluginInference.init();
} catch (std::exception& e)
{
@@ -54,9 +52,9 @@
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
-
-#ifdef TFLITE
- void PluginProcessor::feedInput(const cv::Mat &frame)
+
+#ifdef TFLITE
+ void PluginProcessor::feedInput(const cv::Mat &frame)
{
Plog::log(Plog::LogPriority::INFO, TAG, "inside feedInput()");
auto pair = pluginInference.getInput();
@@ -91,9 +89,9 @@
computedMask = predictions;
}
- void PluginProcessor::printMask()
+ void PluginProcessor::printMask()
{
- for (size_t i = 0; i < computedMask.size(); i++)
+ for (size_t i = 0; i < computedMask.size(); i++)
{
// Log the predictions
std::ostringstream oss;
@@ -102,8 +100,29 @@
}
}
-void PluginProcessor::drawMaskOnReducedFrame(cv::Mat &frame,
- cv::Mat &frameReduced, std::vector<float>computedMask)
+
+ void copyByLine(uchar* frameData, uchar* applyMaskData, const int lineSize, cv::Size size)
+ {
+ if (3 * size.width == lineSize)
+ {
+ std::memcpy(frameData, applyMaskData, size.height * size.width * 3);;
+ }
+ else
+ {
+ int rows = size.height;
+ int offset = 0;
+ int maskoffset = 0;
+ for (int i = 0; i < rows; i++)
+ {
+ std::memcpy(frameData + offset, applyMaskData + maskoffset, lineSize);
+ offset += lineSize;
+ maskoffset += 3 * size.width;
+ }
+ }
+ }
+
+void PluginProcessor::drawMaskOnFrame(cv::Mat &frame,
+ cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
if (computedMask.empty())
@@ -143,12 +162,12 @@
#else
mFloatMask[i] = 0.;
#endif
- }
+ }
}
- cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
+ cv::Mat maskImg(pluginInference.getImageHeight(), pluginInference.getImageWidth(),
CV_32FC1, mFloatMask.data());
- cv::resize(maskImg, maskImg, cv::Size(maskImg.cols+2*absOFFSETX, maskImg.rows+2*absOFFSETY));
+ cv::resize(maskImg, maskImg, cv::Size(frameReduced.cols+2*absOFFSETX, frameReduced.rows+2*absOFFSETY));
kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
if(kSize.height%2 == 0)
@@ -162,12 +181,13 @@
GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
maskImg = maskImg / 255.; //mask from 0 to 1.
-#ifndef __ANDROID__
- cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
+ cv::Mat applyMask = frameReduced.clone();
+
+ cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
-
+
std::vector<cv::Mat> channels;
std::vector<cv::Mat> channelsComplementary;
@@ -181,146 +201,19 @@
cv::merge(channels, roiMaskImg);
cv::merge(channelsComplementary, roiMaskImgComplementary);
-
int origType = frameReduced.type();
int roiMaskType = roiMaskImg.type();
- cv::Mat clone = frameReduced.clone();
-
- clone.convertTo(clone, roiMaskType);
- clone = clone.mul(roiMaskImg);
- clone += backgroundImage.mul(roiMaskImgComplementary);
- clone.convertTo(clone, origType);
- int numberChannels = 3;
+ applyMask.convertTo(applyMask, roiMaskType);
+ applyMask = applyMask.mul(roiMaskImg);
+ applyMask += backgroundImage.mul(roiMaskImgComplementary);
+ applyMask.convertTo(applyMask, origType);
- cv::resize(clone, clone, cv::Size(frame.cols, frame.rows));
+ cv::resize(applyMask, applyMask, cv::Size(frame.cols, frame.rows));
- std::memcpy(frame.data, clone.data,
- static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
-
-#else
- for (int col = 0; col < frame.cols; col++)
- {
- for (int row = 0; row < frame.rows; row++)
- {
- float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
- frame.at<cv::Vec3b>(cv::Point(col, row)) =
- backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
- + frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
- }
- }
-#endif // __ANDROID__
+ copyByLine(frame.data, applyMask.data, lineSize, cv::Size(frame.cols, frame.rows));
computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
}
-
-
- void PluginProcessor::drawMaskOnFrame(
- cv::Mat &frame, std::vector<float>computedMask)
- {
- // Plog::log(Plog::LogPriority::INFO, TAG, "inside drawMaskOnFrame()");
- if (computedMask.empty())
- {
- return;
- }
-
- scaleX = (float)(backgroundImage.cols) / (float)(pluginInference.getImageWidth());
- scaleY = (float)(backgroundImage.rows) / (float)(pluginInference.getImageHeight());
- int absOFFSETY = 4*scaleY;
- int absOFFSETX = 4*scaleX;
- int OFFSETY = -absOFFSETY;
- int OFFSETX = -absOFFSETX;
- if (computedMask1.empty())
- {
- computedMask3 = std::vector<float>(computedMask.size(), 0);
- computedMask2 = std::vector<float>(computedMask.size(), 0);
- computedMask1 = std::vector<float>(computedMask.size(), 0);
- }
-
- std::vector<float> mFloatMask(computedMask.begin(), computedMask.end());
- for (size_t i = 0; i < computedMask.size(); i++)
- {
- if(computedMask[i] == 15)
- {
- computedMask[i] = 255;
- mFloatMask[i] = 255;
- }
- else
- {
- computedMask[i] = 0;
- #ifdef __ANDROID__
- mFloatMask[i] = (float)( (int)((0.6 * computedMask1[i] + 0.3 * computedMask2[i] + 0.1 * computedMask3[i])) % 256 );
- #else
- mFloatMask[i] = 0.;
- #endif
- }
- }
- cv::Mat maskImg(pluginInference.getImageWidth(), pluginInference.getImageHeight(),
- CV_32FC1, mFloatMask.data());
-
- cv::resize(maskImg, maskImg, cv::Size(backgroundImage.cols+2*absOFFSETX, backgroundImage.rows+2*absOFFSETY));
-
- kSize = cv::Size(maskImg.cols*0.05, maskImg.rows*0.05);
- if(kSize.height%2 == 0)
- {
- kSize.height -= 1;
- }
- if(kSize.width%2 == 0)
- {
- kSize.width -= 1;
- }
-
- GaussianBlur (maskImg, maskImg, kSize, 0); //mask from 0 to 255.
- maskImg = maskImg / 255.; //mask from 0 to 1.
-#ifndef __ANDROID__
- cv::Rect roi(absOFFSETX+OFFSETX, absOFFSETY+OFFSETY, backgroundImage.cols, backgroundImage.rows); //Create a rect
- cv::Mat roiMaskImg = maskImg(roi); //Crop the region of interest using above rect
-
- cv::Mat roiMaskImgComplementary = 1. - roiMaskImg; //mask from 1. to 0
-
- std::vector<cv::Mat> channels;
- std::vector<cv::Mat> channelsComplementary;
-
- channels.emplace_back(roiMaskImg);
- channels.emplace_back(roiMaskImg);
- channels.emplace_back(roiMaskImg);
- channelsComplementary.emplace_back(roiMaskImgComplementary);
- channelsComplementary.emplace_back(roiMaskImgComplementary);
- channelsComplementary.emplace_back(roiMaskImgComplementary);
-
- cv::merge(channels, roiMaskImg);
- cv::merge(channelsComplementary, roiMaskImgComplementary);
-
-
- int origType = frame.type();
- int roiMaskType = roiMaskImg.type();
-
- cv::Mat clone = frame.clone();
-
- clone.convertTo(clone, roiMaskType);
- clone = clone.mul(roiMaskImg);
- clone += backgroundImage.mul(roiMaskImgComplementary);
- clone.convertTo(clone, origType);
- int numberChannels = 3;
- std::memcpy(frame.data, clone.data,
- static_cast<size_t>(clone.cols) * static_cast<size_t>(clone.rows) * static_cast<size_t>(numberChannels) * sizeof(uint8_t));
-
-#else
- for (int col = 0; col < frame.cols; col++)
- {
- for (int row = 0; row < frame.rows; row++)
- {
- float maskValue = maskImg.at<float>(cv::Point(col+absOFFSETX+OFFSETX, row+absOFFSETY+OFFSETY));
- frame.at<cv::Vec3b>(cv::Point(col, row)) =
- backgroundImage.at<cv::Vec3b>(cv::Point(col, row)) * (1. - maskValue)
- + frame.at<cv::Vec3b>(cv::Point(col, row)) * maskValue;
- }
- }
-#endif // __ANDROID__
- computedMask3 = std::vector<float>(computedMask2.begin(), computedMask2.end());
- computedMask2 = std::vector<float>(computedMask1.begin(), computedMask1.end());
- computedMask1 = std::vector<float>(computedMask.begin(), computedMask.end());
- }
-
} // namespace jami
\ No newline at end of file
diff --git a/ForegroundSegmentation/pluginProcessor.h b/ForegroundSegmentation/pluginProcessor.h
index 9bdac86..9110754 100644
--- a/ForegroundSegmentation/pluginProcessor.h
+++ b/ForegroundSegmentation/pluginProcessor.h
@@ -19,12 +19,12 @@
// Frame scaler for frame transformations
#include <framescaler.h>
-namespace jami
+namespace jami
{
- class PluginProcessor
+ class PluginProcessor
{
public:
- PluginProcessor(const std::string &dataPath);
+ PluginProcessor(const std::string &dataPath);
//~PluginProcessor();
void initModel();
@@ -41,10 +41,9 @@
* computedPredictions
*/
void computePredictions();
-
+
void printMask();
- void drawMaskOnFrame(cv::Mat &frame, const std::vector<float> computedMask);
- void drawMaskOnReducedFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask);
+ void drawMaskOnFrame(cv::Mat &frame, cv::Mat &frameReduced, std::vector<float>computedMask, int lineSize);
// Output predictions
std::vector<float> computedMask;
@@ -52,14 +51,14 @@
std::vector<float> computedMask2;
std::vector<float> computedMask3;
- cv::Mat backgroundImage;
-
+ cv::Mat backgroundImage;
+
cv::Size kSize;
float scaleX = 0;
- float scaleY = 0;
-
+ float scaleY = 0;
+
PluginInference pluginInference;
- std::string backgroundPath;
+ std::string backgroundPath;
private:
// Frame
diff --git a/ForegroundSegmentation/videoSubscriber.cpp b/ForegroundSegmentation/videoSubscriber.cpp
index d22b977..77659fd 100644
--- a/ForegroundSegmentation/videoSubscriber.cpp
+++ b/ForegroundSegmentation/videoSubscriber.cpp
@@ -14,7 +14,7 @@
const std::string TAG = "FORESEG";
const char sep = separator();
-namespace jami
+namespace jami
{
VideoSubscriber::VideoSubscriber(const std::string &dataPath): path_{dataPath},
pluginProcessor{dataPath}
@@ -23,13 +23,13 @@
* Waits for new frames and then process them
* Writes the predictions in computedPredictions
**/
- processFrameThread = std::thread([this]
+ processFrameThread = std::thread([this]
{
- while (running)
+ while (running)
{
std::unique_lock<std::mutex> l(inputLock);
inputCv.wait(l, [this] { return not running or newFrame; });
- if (not running)
+ if (not running)
{
break;
}
@@ -55,19 +55,19 @@
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
- void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame)
+ void VideoSubscriber::update(jami::Observable<AVFrame *> *, AVFrame *const &iFrame)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "inside update()");
- if (isAttached)
+ if (isAttached)
{
std::ostringstream oss;
//======================================================================================
// GET FRAME ROTATION
AVFrameSideData *side_data =
av_frame_get_side_data(iFrame, AV_FRAME_DATA_DISPLAYMATRIX);
-
+
int angle{0};
- if (side_data)
+ if (side_data)
{
auto matrix_rotation = reinterpret_cast<int32_t *>(side_data->data);
angle = static_cast<int>(av_display_rotation_get(matrix_rotation));
@@ -95,8 +95,8 @@
// ROTATE THE FRAME
// rotateFrame(angle, clone);
// rotateFrame(angle, frame);
-
- if (firstRun)
+
+ if (firstRun)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "step firstRun");
pluginProcessor.pluginInference.setExpectedImageDimensions();
@@ -105,13 +105,13 @@
cv::resize(clone, fcopy.resizedFrameRGB, fcopy.resizedSize);
// cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.originalSize);
cv::resize(pluginProcessor.backgroundImage, pluginProcessor.backgroundImage, fcopy.resizedSize);
-
+
firstRun = false;
}
auto process_start = std::chrono::system_clock::now();
- if (!newFrame)
+ if (!newFrame)
{
// Plog::log(Plog::LogPriority::INFO, TAG, "step newFrame");
std::lock_guard<std::mutex> l(inputLock);
@@ -123,8 +123,8 @@
// Plog::log(Plog::LogPriority::INFO, TAG, "step result");
fcopy.predictionsFrameBGR = frame;
fcopy.predictionsResizedFrameBGR = fcopy.resizedFrameRGB.clone();
- // pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, pluginProcessor.computedMask);
- pluginProcessor.drawMaskOnReducedFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR, pluginProcessor.computedMask);
+ pluginProcessor.drawMaskOnFrame(fcopy.predictionsFrameBGR, fcopy.predictionsResizedFrameBGR,
+ pluginProcessor.computedMask, bgrFrame->linesize[0]);
//======================================================================================
// REPLACE AVFRAME DATA WITH FRAME DATA
@@ -133,26 +133,16 @@
// rotateFrame(-angle, frame);
// Plog::log(Plog::LogPriority::INFO, TAG, "step REPLACE AVFRAME DATA WITH FRAME DATA");
- if (bgrFrame && bgrFrame->data[0])
+ if (bgrFrame && bgrFrame->data[0])
{
uint8_t* frameData = bgrFrame->data[0];
- if(angle == 90 || angle == -90)
+ if(angle == 90 || angle == -90)
{
std::memmove(frameData, fcopy.predictionsFrameBGR.data, static_cast<size_t>(iFrame->width*iFrame->height*3) * sizeof(uint8_t));
}
}
-
- // Plog::log(Plog::LogPriority::INFO, TAG, "step Copy Frame meta data");
- // if (bgrFrame) {
- // Plog::log(Plog::LogPriority::INFO, TAG, "step bgrFrame");
-
- // }
- // if (incFrame) {
- // Plog::log(Plog::LogPriority::INFO, TAG, "step incFrame");
-
- // }
// Copy Frame meta data
- if (bgrFrame && incFrame)
+ if (bgrFrame && incFrame)
{
av_frame_copy_props(bgrFrame.get(), incFrame);
scaler.moveFrom(incFrame, bgrFrame.get());
@@ -171,7 +161,7 @@
}
}
- void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable)
+ void VideoSubscriber::attached(jami::Observable<AVFrame *> *observable)
{
std::ostringstream oss;
oss << "::Attached ! " << std::endl;
@@ -189,7 +179,7 @@
Plog::log(Plog::LogPriority::INFO, TAG, oss.str());
}
- void VideoSubscriber::detach()
+ void VideoSubscriber::detach()
{
if (isAttached)
{
@@ -206,11 +196,11 @@
inputCv.notify_all();
}
- void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat)
+ void VideoSubscriber::rotateFrame(int angle, cv::Mat &mat)
{
- if (angle != 0)
+ if (angle != 0)
{
- switch (angle)
+ switch (angle)
{
case -90:
cv::rotate(mat, mat, cv::ROTATE_90_COUNTERCLOCKWISE);
@@ -226,4 +216,3 @@
}
}
}
-