原理概述
代码
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/core/core.hpp"
#include <assert.h>
#include <stdio.h>
#include <algorithm>
#include <vector>
#include <string>
#include <iostream>
#include <cmath>
#include <iostream>
#include <string>
#include <opencv2\imgproc\types_c.h>
using namespace cv;
using namespace std;
typedef std::vector<std::vector<cv::Point>> contours_t;
typedef std::vector<cv::Vec4i> hierarchy_t;
typedef std::vector<cv::Point> contour_t;
// Patch raduius
#define RADIUS 4
// The maximum number of pixels around a specified point on the target outline
#define BORDER_RADIUS 4
int mod(int a, int b);
void loadInpaintingImages(
const std::string& colorFilename,
const std::string& maskFilename,
cv::Mat& colorMat,
cv::Mat& maskMat,
cv::Mat& grayMat);
void getContours(const cv::Mat& mask, contours_t& contours, hierarchy_t& hierarchy);
double computeConfidence(const cv::Mat& confidencePatch);
cv::Mat getPatch(const cv::Mat& image, const cv::Point& p);
void getDerivatives(const cv::Mat& grayMat, cv::Mat& dx, cv::Mat& dy);
cv::Point2f getNormal(const contour_t& contour, const cv::Point& point);
void computePriority(const contours_t& contours, const cv::Mat& grayMat, const cv::Mat& confidenceMat, cv::Mat& priorityMat);
void transferPatch(const cv::Point& psiHatQ, const cv::Point& psiHatP, cv::Mat& mat, const cv::Mat& maskMat);
cv::Mat computeSSD(const cv::Mat& tmplate, const cv::Mat& source, const cv::Mat& tmplateMask);
#ifndef DEBUG
#define DEBUG 0
#endif
/*
* Return a % b where % is the mathematical modulus operator.
*/
int mod(int a, int b)
{
return ((a % b) + b) % b;
}
/*
* Load the color, mask, grayscale images with a border of size
* radius around every image to prevent boundary collisions when taking patches
*/
void loadInpaintingImages(
const std::string& colorFilename,
const std::string& maskFilename,
cv::Mat& colorMat,
cv::Mat& maskMat,
cv::Mat& grayMat)
{
assert(colorFilename.length() && maskFilename.length());
colorMat = cv::imread(colorFilename, 1); // color
maskMat = cv::imread(maskFilename, 0); // grayscale
assert(colorMat.size() == maskMat.size());
assert(!colorMat.empty() && !maskMat.empty());
// convert colorMat to depth CV_32F for colorspace conversions
colorMat.convertTo(colorMat, CV_32F);
colorMat /= 255.0f;
// add border around colorMat
cv::copyMakeBorder(
colorMat,
colorMat,
RADIUS,
RADIUS,
RADIUS,
RADIUS,
cv::BORDER_CONSTANT,
cv::Scalar_<float>(0, 0, 0)
);
cv::cvtColor(colorMat, grayMat, CV_BGR2GRAY);
}
/*
* Extract closed boundary from mask.
*/
void getContours(const cv::Mat& mask,
contours_t& contours,
hierarchy_t& hierarchy
)
{
assert(mask.type() == CV_8UC1);
cv::findContours(mask, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);
}
/*
* Get a patch of size RAIDUS around point p in mat.
*/
cv::Mat getPatch(const cv::Mat& mat, const cv::Point& p)
{
assert(RADIUS <= p.x && p.x < mat.cols - RADIUS && RADIUS <= p.y && p.y < mat.rows - RADIUS);
return mat(
cv::Range(p.y - RADIUS, p.y + RADIUS + 1),
cv::Range(p.x - RADIUS, p.x + RADIUS + 1)
);
}
// get the x and y derivatives of a patch centered at patchCenter in image
// computed using a 3x3 Scharr filter
void getDerivatives(const cv::Mat& grayMat, cv::Mat& dx, cv::Mat& dy)
{
assert(grayMat.type() == CV_32FC1);
cv::Sobel(grayMat, dx, -1, 1, 0, -1);
cv::Sobel(grayMat, dy, -1, 0, 1, -1);
}
/*
* Get the unit normal of a dense list of boundary points centered around point p.
*/
cv::Point2f getNormal(const contour_t& contour, const cv::Point& point)
{
int sz = (int)contour.size();
assert(sz != 0);
int pointIndex = (int)(std::find(contour.begin(), contour.end(), point) - contour.begin());
assert(pointIndex != contour.size());
if (sz == 1)
{
return cv::Point2f(1.0f, 0.0f);
}
else if (sz < 2 * BORDER_RADIUS + 1)
{
// Too few points in contour to use LSTSQ regression
// return the normal with respect to adjacent neigbourhood
cv::Point adj = contour[(pointIndex + 1) % sz] - contour[pointIndex];
return cv::Point2f(adj.y, -adj.x) / cv::norm(adj);
}
// Use least square regression
// create X and Y mat to SVD
cv::Mat X(cv::Size(2, 2 * BORDER_RADIUS + 1), CV_32F);
cv::Mat Y(cv::Size(1, 2 * BORDER_RADIUS + 1), CV_32F);
assert(X.rows == Y.rows && X.cols == 2 && Y.cols == 1 && X.type() == Y.type()
&& Y.type() == CV_32F);
int i = mod((pointIndex - BORDER_RADIUS), sz);
float* Xrow;
float* Yrow;
int count = 0;
int countXequal = 0;
while (count < 2 * BORDER_RADIUS + 1)
{
Xrow = X.ptr<float>(count);
Xrow[0] = contour[i].x;
Xrow[1] = 1.0f;
Yrow = Y.ptr<float>(count);
Yrow[0] = contour[i].y;
if (Xrow[0] == contour[pointIndex].x)
{
++countXequal;
}
i = mod(i + 1, sz);
++count;
}
if (countXequal == count)
{
return cv::Point2f(1.0f, 0.0f);
}
// to find the line of best fit
cv::Mat sol;
cv::solve(X, Y, sol, cv::DECOMP_SVD);
assert(sol.type() == CV_32F);
float slope = sol.ptr<float>(0)[0];
cv::Point2f normal(-slope, 1);
return normal / cv::norm(normal);
}
/*
* Return the confidence of confidencePatch
*/
double computeConfidence(const cv::Mat& confidencePatch)
{
return cv::sum(confidencePatch)[0] / (double)confidencePatch.total();
}
/*
* Iterate over every contour point in contours and compute the
* priority of path centered at point using grayMat and confidenceMat
*/
void computePriority(const contours_t& contours, const cv::Mat& grayMat, const cv::Mat& confidenceMat, cv::Mat& priorityMat)
{
assert(grayMat.type() == CV_32FC1 &&
priorityMat.type() == CV_32FC1 &&
confidenceMat.type() == CV_32FC1
);
// define some patches
cv::Mat confidencePatch;
cv::Mat magnitudePatch;
cv::Point2f normal;
cv::Point maxPoint;
cv::Point2f gradient;
double confidence;
// get the derivatives and magnitude of the greyscale image
cv::Mat dx, dy, magnitude;
getDerivatives(grayMat, dx, dy);
cv::magnitude(dx, dy, magnitude);
// mask the magnitude
cv::Mat maskedMagnitude(magnitude.size(), magnitude.type(), cv::Scalar_<float>(0));
magnitude.copyTo(maskedMagnitude, (confidenceMat != 0.0f));
cv::erode(maskedMagnitude, maskedMagnitude, cv::Mat());
assert(maskedMagnitude.type() == CV_32FC1);
// for each point in contour
cv::Point point;
for (int i = 0; i < contours.size(); ++i)
{
contour_t contour = contours[i];
for (int j = 0; j < contour.size(); ++j)
{
point = contour[j];
confidencePatch = getPatch(confidenceMat, point);
// get confidence of patch
confidence = cv::sum(confidencePatch)[0] / (double)confidencePatch.total();
assert(0 <= confidence && confidence <= 1.0f);
// get the normal to the border around point
normal = getNormal(contour, point);
// get the maximum gradient in source around patch
magnitudePatch = getPatch(maskedMagnitude, point);
cv::minMaxLoc(magnitudePatch, NULL, NULL, NULL, &maxPoint);
gradient = cv::Point2f(
-getPatch(dy, point).ptr<float>(maxPoint.y)[maxPoint.x],
getPatch(dx, point).ptr<float>(maxPoint.y)[maxPoint.x]
);
// set the priority in priorityMat
priorityMat.ptr<float>(point.y)[point.x] = std::abs((float)confidence * gradient.dot(normal));
assert(priorityMat.ptr<float>(point.y)[point.x] >= 0);
}
}
}
/*
* Transfer the values from patch centered at psiHatQ to patch centered at psiHatP in
* mat according to maskMat.
*/
void transferPatch(const cv::Point& psiHatQ, const cv::Point& psiHatP, cv::Mat& mat, const cv::Mat& maskMat)
{
assert(maskMat.type() == CV_8U);
assert(mat.size() == maskMat.size());
assert(RADIUS <= psiHatQ.x && psiHatQ.x < mat.cols - RADIUS && RADIUS <= psiHatQ.y && psiHatQ.y < mat.rows - RADIUS);
assert(RADIUS <= psiHatP.x && psiHatP.x < mat.cols - RADIUS && RADIUS <= psiHatP.y && psiHatP.y < mat.rows - RADIUS);
// copy contents of psiHatQ to psiHatP with mask
getPatch(mat, psiHatQ).copyTo(getPatch(mat, psiHatP), getPatch(maskMat, psiHatP));
}
/*
* Runs template matching with tmplate and mask tmplateMask on source.
* Resulting Mat is stored in result.
*
*/
cv::Mat computeSSD(const cv::Mat& tmplate, const cv::Mat& source, const cv::Mat& tmplateMask)
{
assert(tmplate.type() == CV_32FC3 && source.type() == CV_32FC3);
assert(tmplate.rows <= source.rows && tmplate.cols <= source.cols);
assert(tmplateMask.size() == tmplate.size() && tmplate.type() == tmplateMask.type());
cv::Mat result(source.rows - tmplate.rows + 1, source.cols - tmplate.cols + 1, CV_32F, 0.0f);
cv::matchTemplate(source,
tmplate,
result,
CV_TM_SQDIFF,
tmplateMask
);
cv::normalize(result, result, 0, 1, cv::NORM_MINMAX);
cv::copyMakeBorder(result, result, RADIUS, RADIUS, RADIUS, RADIUS, cv::BORDER_CONSTANT, 0.1f);
return result;
}
int main (int argc, char** argv) {
// --------------- read filename strings ------------------
std::string colorFilename, maskFilename;
colorFilename = "criminisi1.bmp";
maskFilename = "criminisi1_mask.bmp";
// ---------------- read the images ------------------------
// colorMat - color picture + border
// maskMat - mask picture + border
// grayMat - gray picture + border
cv::Mat colorMat, maskMat, grayMat;
loadInpaintingImages(
colorFilename,
maskFilename,
colorMat,
maskMat,
grayMat
);
// confidenceMat - confidence picture + border
cv::Mat confidenceMat;
maskMat.convertTo(confidenceMat, CV_32F);
confidenceMat /= 255.0f;
// add borders around maskMat and confidenceMat
cv::copyMakeBorder(maskMat, maskMat,
RADIUS, RADIUS, RADIUS, RADIUS,
cv::BORDER_CONSTANT, 255);
cv::copyMakeBorder(confidenceMat, confidenceMat,
RADIUS, RADIUS, RADIUS, RADIUS,
cv::BORDER_CONSTANT, 0.0001f);
// ---------------- start the algorithm -----------------
contours_t contours; // mask contours
hierarchy_t hierarchy; // contours hierarchy
// priorityMat - priority values for all contour points + border
cv::Mat priorityMat(
confidenceMat.size(),
CV_32FC1
); // priority value matrix for each contour point
assert(
colorMat.size() == grayMat.size() &&
colorMat.size() == confidenceMat.size() &&
colorMat.size() == maskMat.size()
);
cv::Point psiHatP; // psiHatP - point of highest confidence
cv::Mat psiHatPColor; // color patch around psiHatP
cv::Mat psiHatPConfidence; // confidence patch around psiHatP
double confidence; // confidence of psiHatPConfidence
cv::Point psiHatQ; // psiHatQ - point of closest patch
cv::Mat result; // holds result from template matching
cv::Mat erodedMask; // eroded mask
cv::Mat templateMask; // mask for template match (3 channel)
// eroded mask is used to ensure that psiHatQ is not overlapping with target
cv::erode(maskMat, erodedMask, cv::Mat(), cv::Point(-1, -1), RADIUS);
cv::Mat drawMat;
// main loop
const size_t area = maskMat.total();
while (cv::countNonZero(maskMat) != area) // end when target is filled
{
// set priority matrix to -.1, lower than 0 so that border area is never selected
priorityMat.setTo(-0.1f);
// get the contours of mask
getContours((maskMat == 0), contours, hierarchy);
if (DEBUG) {
drawMat = colorMat.clone();
}
// compute the priority for all contour points
computePriority(contours, grayMat, confidenceMat, priorityMat);
// get the patch with the greatest priority
cv::minMaxLoc(priorityMat, NULL, NULL, NULL, &psiHatP);
psiHatPColor = getPatch(colorMat, psiHatP);
psiHatPConfidence = getPatch(confidenceMat, psiHatP);
cv::Mat confInv = (psiHatPConfidence != 0.0f);
confInv.convertTo(confInv, CV_32F);
confInv /= 255.0f;
// get the patch in source with least distance to psiHatPColor wrt source of psiHatP
cv::Mat mergeArrays[3] = {confInv, confInv, confInv};
cv::merge(mergeArrays, 3, templateMask);
result = computeSSD(psiHatPColor, colorMat, templateMask);
// set all target regions to 1.1, which is over the maximum value possilbe
// from SSD
result.setTo(1.1f, erodedMask == 0);
// get minimum point of SSD between psiHatPColor and colorMat
cv::minMaxLoc(result, NULL, NULL, &psiHatQ);
assert(psiHatQ != psiHatP);
if (DEBUG) {
cv::rectangle(drawMat, psiHatP - cv::Point(RADIUS, RADIUS), psiHatP + cv::Point(RADIUS+1, RADIUS+1), cv::Scalar(255, 0, 0));
cv::rectangle(drawMat, psiHatQ - cv::Point(RADIUS, RADIUS), psiHatQ + cv::Point(RADIUS+1, RADIUS+1), cv::Scalar(0, 0, 255));
imshow("red - psiHatQ", drawMat);
}
// updates
// copy from psiHatQ to psiHatP for each colorspace
transferPatch(psiHatQ, psiHatP, grayMat, (maskMat == 0));
transferPatch(psiHatQ, psiHatP, colorMat, (maskMat == 0));
// fill in confidenceMat with confidences C(pixel) = C(psiHatP)
confidence = computeConfidence(psiHatPConfidence);
assert(0 <= confidence && confidence <= 1.0f);
// update confidence
psiHatPConfidence.setTo(confidence, (psiHatPConfidence == 0.0f));
// update maskMat
maskMat = (confidenceMat != 0.0f);
}
//showMat("final result", colorMat, 0);
Mat imageROI, image_src;//ROI区域
int x_begin, y_begin, width, height; //裁取区域的坐标及大小
int srcWidth, srcHeight; //存储原图宽、高
srcWidth = colorMat.cols; //获取原图宽、高
srcHeight = colorMat.rows;
x_begin = 4;
y_begin = 4;
width = srcWidth - 8;
height = srcHeight - 8;
image_src = colorMat.clone(); //备份原图
imageROI = image_src(Rect(x_begin, y_begin, width, height)); //设置待裁取ROI
imshow("final result", imageROI);
normalize(imageROI, imageROI, 0, 255, NORM_MINMAX, CV_8U);
imwrite("result.jpg", imageROI);
waitKey(0);
return 0;
}
测试图像1
掩模图像1
结果图像1
测试图像2
掩模图像2
结果图像2
测试图像3
掩模图像3
结果图像3
测试图像4
掩模图像4
结果图像4
