uvos/SDImagePreprocess
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

paralleization wip

Browse Source
This commit is contained in:
uvos
2023-06-30 00:48:56 +02:00
parent f5dad284e6
commit b3c2d585ae
5 changed files with 230 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
intelligentroi.cpp
View File

@ -31,11 +31,12 @@ void InteligentRoi::slideRectToPoint(cv::Rect& rect, const cv::Point2i& point)
}
}
cv::Rect InteligentRoi::maxRect(const cv::Size2i& imageSize, std::vector<std::pair<cv::Point2i, int>> mustInclude)
cv::Rect InteligentRoi::maxRect(bool& incompleate, const cv::Size2i& imageSize, std::vector<std::pair<cv::Point2i, int>> mustInclude)
{
int radius = std::min(imageSize.height, imageSize.width)/2;
incompleate = false;
int diameter = std::min(imageSize.height, imageSize.width);
cv::Point2i point(imageSize.width/2, imageSize.height/2);
cv::Rect candiate(point.x-radius, point.y-radius, radius*2, radius*2);
cv::Rect candiate(point.x-diameter/2, point.y-diameter/2, diameter, diameter);
std::sort(mustInclude.begin(), mustInclude.end(),
[&point](const std::pair<cv::Point2i, int>& a, const std::pair<cv::Point2i, int>& b){return compPointPrio(a, b, point);});
@ -43,8 +44,9 @@ cv::Rect InteligentRoi::maxRect(const cv::Size2i& imageSize, std::vector<std::pa
while(true)
{
cv::Rect includeRect = rectFromPoints(mustInclude);
if(includeRect.width-2 > radius || includeRect.height-2 > radius)
if(includeRect.width-2 > diameter || includeRect.height-2 > diameter)
{
incompleate = true;
slideRectToPoint(candiate, mustInclude.back().first);
mustInclude.pop_back();
Log(Log::DEBUG)<<"cant fill";
@ -52,8 +54,10 @@ cv::Rect InteligentRoi::maxRect(const cv::Size2i& imageSize, std::vector<std::pa
Log(Log::DEBUG)<<mipoint.first<<' '<<pointDist(mipoint.first, point)<<' '<<mipoint.second;
}
else
{
break;
}
}
for(const std::pair<cv::Point2i, int>& includePoint : mustInclude)
slideRectToPoint(candiate, includePoint.first);
@ -75,9 +79,7 @@ InteligentRoi::InteligentRoi(const Yolo& yolo)
personId = yolo.getClassForStr("person");
}
cv::Rect InteligentRoi::getCropRectangle(const std::vector<Yolo::Detection>& detections, const cv::Size2i& imageSize)
{
if(!detections.empty())
bool InteligentRoi::getCropRectangle(cv::Rect& out, const std::vector<Yolo::Detection>& detections, const cv::Size2i& imageSize)
{
std::vector<std::pair<cv::Point2i, int>> corners;
for(size_t i = 0; i < detections.size(); ++i)
@ -100,9 +102,7 @@ cv::Rect InteligentRoi::getCropRectangle(const std::vector<Yolo::Detection>& det
}
}
return maxRect(imageSize, corners);
}
Log(Log::DEBUG)<<"Using center crop as there are no detections";
return maxRect(imageSize);
bool incompleate;
out = maxRect(incompleate, imageSize, corners);
return incompleate;
}

4
intelligentroi.h
View File

@ -10,9 +10,9 @@ private:
int personId;
static bool compPointPrio(const std::pair<cv::Point2i, int>& a, const std::pair<cv::Point2i, int>& b, const cv::Point2i& center);
static void slideRectToPoint(cv::Rect& rect, const cv::Point2i& point);
static cv::Rect maxRect(const cv::Size2i& imageSize, std::vector<std::pair<cv::Point2i, int>> mustInclude = {});
static cv::Rect maxRect(bool& incompleate, const cv::Size2i& imageSize, std::vector<std::pair<cv::Point2i, int>> mustInclude = {});
public:
InteligentRoi(const Yolo& yolo);
cv::Rect getCropRectangle(const std::vector<Yolo::Detection>& detections, const cv::Size2i& imageSize);
bool getCropRectangle(cv::Rect& out, const std::vector<Yolo::Detection>& detections, const cv::Size2i& imageSize);
};

341
main.cpp
View File

@ -1,9 +1,13 @@
#include <filesystem>
#include <iostream>
#include <opencv2/core.hpp>
#include <opencv2/core/types.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <algorithm>
#include <execution>
#include <vector>
#include <numeric>
#include "yolo.h"
#include "log.h"
@ -17,10 +21,10 @@ const Yolo::Detection* pointInDetectionHoriz(int x, const std::vector<Yolo::Dete
const Yolo::Detection* inDetection = nullptr;
for(const Yolo::Detection& detection : detections)
{
if(!ignore || ignore != &detection)
if(ignore && ignore == &detection)
continue;
if(detection.box.x <= x && detection.box.x+detection.box.width <= x)
if(detection.box.x <= x && detection.box.x+detection.box.width >= x)
{
if(!inDetection || detection.box.br().x > inDetection->box.br().x)
inDetection = &detection;
@ -33,6 +37,8 @@ bool findRegionEndpointHoriz(int& x, const std::vector<Yolo::Detection>& detecti
{
const Yolo::Detection* inDetection = pointInDetectionHoriz(x, detections);
Log(Log::DEBUG, false)<<__func__<<" point "<<x;
if(!inDetection)
{
const Yolo::Detection* closest = nullptr;
@ -48,123 +54,51 @@ bool findRegionEndpointHoriz(int& x, const std::vector<Yolo::Detection>& detecti
x = closest->box.x;
else
x = imgSizeX;
Log(Log::DEBUG)<<" is not in any box and will be moved to "<<x<<" where the closest box ("<<(closest ? closest->className : "null")<<") is";
return false;
}
else
{
x = inDetection->box.br().x;
Log(Log::DEBUG, false)<<" is in a box and will be moved to its end "<<x<<" where ";
const Yolo::Detection* candidateDetection = pointInDetectionHoriz(x, detections, inDetection);
if(candidateDetection && candidateDetection->box.br().x > x)
{
Log(Log::DEBUG)<<"it is again in a box";
return findRegionEndpointHoriz(x, detections, imgSizeX);
}
else
{
Log(Log::DEBUG)<<"it is not in a box";
return true;
}
}
}
std::vector<std::pair<cv::Mat, bool>> cutImageIntoHorzRegions(cv::Mat& image, const std::vector<Yolo::Detection>& detections)
{
std::vector<std::pair<cv::Mat, bool>> out;
std::cout<<__func__<<' '<<image.cols<<'x'<<image.rows<<std::endl;
for(int x = 0; x < image.cols; ++x)
{
int start = x;
bool frozen = findRegionEndpointHoriz(x, detections, image.cols);
cv::Mat slice = image(cv::Rect(start, 0, x-start, image.rows));
int width = x-start;
if(x < image.cols)
++width;
cv::Rect rect(start, 0, width, image.rows);
Log(Log::DEBUG)<<__func__<<" region\t"<<rect;
cv::Mat slice = image(rect);
out.push_back({slice, frozen});
}
return out;
}
const Yolo::Detection* pointInDetectionVert(int y, const std::vector<Yolo::Detection>& detections, const Yolo::Detection* ignore = nullptr)
{
const Yolo::Detection* inDetection = nullptr;
for(const Yolo::Detection& detection : detections)
{
if(!ignore || ignore != &detection)
continue;
if(detection.box.y <= y && detection.box.y+detection.box.height <= y)
{
if(!inDetection || detection.box.br().y > inDetection->box.br().y)
inDetection = &detection;
}
}
return inDetection;
}
bool findRegionEndpointVert(int& y, const std::vector<Yolo::Detection>& detections, int imgSizeY)
{
const Yolo::Detection* inDetection = pointInDetectionVert(y, detections);
if(!inDetection)
{
const Yolo::Detection* closest = nullptr;
for(const Yolo::Detection& detection : detections)
{
if(detection.box.y > y)
{
if(closest == nullptr || detection.box.y-y > closest->box.y-y)
closest = &detection;
}
}
if(closest)
y = closest->box.y;
else
y = imgSizeY;
return false;
}
else
{
y = inDetection->box.br().y;
const Yolo::Detection* candidateDetection = pointInDetectionVert(y, detections, inDetection);
if(candidateDetection && candidateDetection->box.br().y > y)
return findRegionEndpointVert(y, detections, imgSizeY);
else
return true;
}
}
std::vector<std::pair<cv::Mat, bool>> cutImageIntoVertRegions(cv::Mat& image, const std::vector<Yolo::Detection>& detections)
{
std::vector<std::pair<cv::Mat, bool>> out;
for(int y = 0; y < image.rows; ++y)
{
int start = y;
bool frozen = findRegionEndpointVert(y, detections, image.rows);
cv::Mat slice = image(cv::Rect(0, start, image.cols, y-start));
out.push_back({slice, frozen});
}
return out;
}
cv::Mat assembleFromSlicesVert(const std::vector<std::pair<cv::Mat, bool>>& slices)
{
assert(!slices.empty());
int rows = 0;
for(const std::pair<cv::Mat, bool>& slice : slices)
rows += slice.first.rows;
cv::Mat image(slices[0].first.cols, rows, slices[0].first.type());
Log(Log::DEBUG)<<__func__<<" assembled image size "<<image.size;
int row = 0;
for(const std::pair<cv::Mat, bool>& slice : slices)
{
cv::Rect rect(0, row, slice.first.cols, slice.first.rows);
slice.first.copyTo(image(rect));
row += slice.first.rows;
}
return image;
}
cv::Mat assembleFromSlicesHoriz(const std::vector<std::pair<cv::Mat, bool>>& slices)
{
assert(!slices.empty());
@ -173,19 +107,33 @@ cv::Mat assembleFromSlicesHoriz(const std::vector<std::pair<cv::Mat, bool>>& sli
for(const std::pair<cv::Mat, bool>& slice : slices)
cols += slice.first.cols;
cv::Mat image(cols, slices[0].first.rows, slices[0].first.type());
Log(Log::DEBUG)<<__func__<<' '<<image.size()<<' '<<cols<<' '<<slices[0].first.rows;
int col = 0;
for(const std::pair<cv::Mat, bool>& slice : slices)
{
cv::Rect rect(col, 0, slice.first.cols, slice.first.rows);
Log(Log::DEBUG)<<__func__<<' '<<rect;
slice.first.copyTo(image(rect));
col += slice.first.cols;
col += slice.first.cols-1;
}
return image;
}
void transposeRect(cv::Rect& rect)
{
int x = rect.x;
rect.x = rect.y;
rect.y = x;
int width = rect.width;
rect.width = rect.height;
rect.height = width;
}
bool seamCarveResize(cv::Mat& image, std::vector<Yolo::Detection> detections, double targetAspectRatio = 1.0)
{
detections.erase(std::remove_if(detections.begin(), detections.end(), [](const Yolo::Detection& detection){return detection.priority < 3;}), detections.end());
@ -206,66 +154,67 @@ bool seamCarveResize(cv::Mat& image, std::vector<Yolo::Detection> detections, do
Log(Log::DEBUG)<<__func__<<' '<<requiredLines<<" lines are required in "<<(vertical ? "vertical" : "horizontal")<<" direction";
if(!vertical)
if(vertical)
{
cv::transpose(image, image);
for(Yolo::Detection& detection : detections)
transposeRect(detection.box);
}
std::vector<std::pair<cv::Mat, bool>> slices = cutImageIntoHorzRegions(image, detections);
Log(Log::DEBUG)<<"Image has "<<slices.size()<<" slices";
Log(Log::DEBUG)<<"Image has "<<slices.size()<<" slices:";
int totalResizableSize = 0;
for(const std::pair<cv::Mat, bool>& slice : slices)
{
if(slice.second)
Log(Log::DEBUG)<<"a "<<(slice.second ? "frozen" : "unfrozen")<<" slice of size "<<slice.first.cols;
if(!slice.second)
totalResizableSize += slice.first.cols;
}
if(totalResizableSize < requiredLines/10)
if(totalResizableSize < requiredLines+1)
{
Log(Log::WARN)<<"Unable to seam carve as there are only "<<totalResizableSize<<" unfrozen cols";
if(vertical)
cv::transpose(image, image);
return false;
}
std::vector<int> seamsForSlice(slices.size(), 0);
for(size_t i = 0; i < slices.size(); ++i)
{
if(!slices[i].second)
seamsForSlice[i] = (static_cast<double>(slices[i].first.cols)/totalResizableSize)*requiredLines;
}
int residual = requiredLines - std::accumulate(seamsForSlice.begin(), seamsForSlice.end(), decltype(seamsForSlice)::value_type(0));;
for(ssize_t i = slices.size()-1; i >= 0; --i)
{
if(!slices[i].second)
{
seamsForSlice[i] += residual;
break;
}
}
for(size_t i = 0; i < slices.size(); ++i)
{
if(slices[i].second)
if(seamsForSlice[i] != 0)
{
int seamsForSlice = (static_cast<double>(slices[i].first.cols)/totalResizableSize)*requiredLines;
bool ret = SeamCarving::strechImage(image, seamsForSlice, true);
bool ret = SeamCarving::strechImage(slices[i].first, seamsForSlice[i], true);
if(!ret)
{
if(vertical)
transpose(image, image);
return false;
}
}
}
image = assembleFromSlicesHoriz(slices);
}
else
{
std::vector<std::pair<cv::Mat, bool>> slices = cutImageIntoVertRegions(image, detections);
Log(Log::DEBUG)<<"Image has "<<slices.size()<<" slices";
int totalResizableSize = 0;
for(const std::pair<cv::Mat, bool>& slice : slices)
{
if(slice.second)
totalResizableSize += slice.first.rows;
}
if(totalResizableSize < requiredLines/10)
{
Log(Log::WARN)<<"Unable to seam carve as there are only "<<totalResizableSize<<" unfrozen rows";
return false;
}
if(vertical)
cv::transpose(image, image);
for(size_t i = 0; i < slices.size(); ++i)
{
if(slices[i].second)
{
int seamsForSlice = (static_cast<double>(slices[i].first.rows)/totalResizableSize)*requiredLines;
bool ret = SeamCarving::strechImageVert(image, seamsForSlice, true);
if(!ret)
return false;
}
}
image = assembleFromSlicesVert(slices);
}
return true;
}
@ -284,6 +233,85 @@ void drawDebugInfo(cv::Mat &image, const cv::Rect& rect, const std::vector<Yolo:
cv::rectangle(image, rect, cv::Scalar(0, 0, 255), 8);
}
static void reduceSize(cv::Mat& image, const cv::Size& targetSize)
{
int longTargetSize = std::max(targetSize.width, targetSize.height)*2;
if(std::max(image.cols, image.rows) > longTargetSize)
{
if(image.cols > image.rows)
{
double ratio = static_cast<double>(longTargetSize)/image.cols;
cv::resize(image, image, {longTargetSize, static_cast<int>(image.rows*ratio)}, 0, 0, cv::INTER_CUBIC);
}
else
{
double ratio = static_cast<double>(longTargetSize)/image.rows;
cv::resize(image, image, {static_cast<int>(image.cols*ratio), longTargetSize}, 0, 0, cv::INTER_CUBIC);
}
}
}
void pipeline(const std::filesystem::path& path, const Config& config, Yolo& yolo, const std::filesystem::path& debugOutputPath)
{
InteligentRoi intRoi(yolo);
cv::Mat image = cv::imread(path);
if(!image.data)
{
Log(Log::WARN)<<"could not load image "<<path<<" skipping";
return;
}
reduceSize(image, config.targetSize);
std::vector<Yolo::Detection> detections = yolo.runInference(image);
Log(Log::DEBUG)<<"Got "<<detections.size()<<" detections for "<<path;
for(const Yolo::Detection& detection : detections)
Log(Log::DEBUG)<<detection.class_id<<": "<<detection.className<<" at "<<detection.box<<" with prio "<<detection.priority;
cv::Rect crop;
bool incompleate = intRoi.getCropRectangle(crop, detections, image.size());
if(config.seamCarving && incompleate)
{
bool ret = seamCarveResize(image, detections, config.targetSize.aspectRatio());
if(ret && image.size().aspectRatio() != config.targetSize.aspectRatio())
detections = yolo.runInference(image);
}
cv::Mat croppedImage;
if(image.size().aspectRatio() != config.targetSize.aspectRatio() && incompleate)
{
intRoi.getCropRectangle(crop, detections, image.size());
if(config.debug)
{
cv::Mat debugImage = image.clone();
drawDebugInfo(debugImage, crop, detections);
bool ret = cv::imwrite(debugOutputPath/path.filename(), debugImage);
if(!ret)
Log(Log::WARN)<<"could not save debug image to "<<debugOutputPath/path.filename()<<" skipping";
}
croppedImage = image(crop);
}
else if(!incompleate)
{
croppedImage = image(crop);
}
else
{
croppedImage = image;
}
cv::Mat resizedImage;
cv::resize(croppedImage, resizedImage, {512, 512}, 0, 0, cv::INTER_CUBIC);
bool ret = cv::imwrite(config.outputDir/path.filename(), resizedImage);
if(!ret)
Log(Log::WARN)<<"could not save image to "<<config.outputDir/path.filename()<<" skipping";
}
int main(int argc, char* argv[])
{
Log::level = Log::INFO;
@ -319,7 +347,6 @@ int main(int argc, char* argv[])
}
Yolo yolo(config.modelPath, {640, 480}, config.classesPath, false);
InteligentRoi intRoi(yolo);
if(!std::filesystem::exists(config.outputDir))
{
@ -337,60 +364,8 @@ int main(int argc, char* argv[])
std::filesystem::create_directory(debugOutputPath);
}
for(const std::filesystem::path& path : imagePaths)
{
cv::Mat image = cv::imread(path);
if(!image.data)
{
Log(Log::WARN)<<"could not load image "<<path<<" skipping";
continue;
}
std::for_each(std::execution::parallel_unsequenced_policy(),
imagePaths.begin(), imagePaths.end(), [&yolo, &debugOutputPath, &config](const std::filesystem::path& path){pipeline(path, config, yolo, debugOutputPath);});
if(std::max(image.cols, image.rows) > 1024)
{
Log(Log::DEBUG, false)<<"Image is "<<image.size();
if(image.cols > image.rows)
{
double ratio = 1024.0/image.cols;
cv::resize(image, image, {1024, static_cast<int>(image.rows*ratio)}, 0, 0, cv::INTER_CUBIC);
}
else
{
double ratio = 1024.0/image.rows;
cv::resize(image, image, {static_cast<int>(image.cols*ratio), 1024}, 0, 0, cv::INTER_CUBIC);
}
Log(Log::DEBUG)<<" resized to "<<image.size();
}
std::vector<Yolo::Detection> detections = yolo.runInference(image);
Log(Log::DEBUG)<<"Got "<<detections.size()<<" detections for "<<path;
for(const Yolo::Detection& detection : detections)
Log(Log::DEBUG)<<detection.class_id<<": "<<detection.className<<" at "<<detection.box<<" with prio "<<detection.priority;
if(config.seamCarving)
{
Log(Log::INFO)<<"Trying seam resize";
seamCarveResize(image, detections, 1);
}
cv::Rect crop = intRoi.getCropRectangle(detections, image.size());
if(config.debug)
{
cv::Mat debugImage = image.clone();
drawDebugInfo(debugImage, crop, detections);
bool ret = cv::imwrite(debugOutputPath/path.filename(), debugImage);
if(!ret)
Log(Log::WARN)<<"could not save debug image to "<<debugOutputPath/path.filename()<<" skipping";
}
cv::Mat croppedImage = image(crop);
cv::Mat resizedImage;
cv::resize(croppedImage, resizedImage, {512, 512}, 0, 0, cv::INTER_CUBIC);
bool ret = cv::imwrite(config.outputDir/path.filename(), resizedImage);
if(!ret)
Log(Log::WARN)<<"could not save image to "<<config.outputDir/path.filename()<<" skipping";
}
return 0;
}

5
options.h
View File

@ -4,6 +4,7 @@
#include <argp.h>
#include <iostream>
#include <filesystem>
#include <opencv2/core/types.hpp>
#include "log.h"
const char *argp_program_version = "AIImagePreprocesses";
@ -19,7 +20,8 @@ static struct argp_option options[] =
{"classes", 'c', "[FILENAME]", 0, "classes text file to use" },
{"out", 'o', "[DIRECTORY]", 0, "directory whre images are to be saved" },
{"debug", 'd', 0, 0, "output debug images" },
{"seam-carving", 's', 0, 0, "model to train"}
{"seam-carving", 's', 0, 0, "model to train"},
{0}
};
struct Config
@ -30,6 +32,7 @@ struct Config
std::filesystem::path outputDir;
bool seamCarving = false;
bool debug = false;
cv::Size targetSize = cv::Size(512, 512);
};
static error_t parse_opt (int key, char *arg, struct argp_state *state)

35
seamcarving.cpp
View File

@ -11,38 +11,53 @@
bool SeamCarving::strechImage(cv::Mat& image, int seams, bool grow, std::vector<std::vector<int>>* seamsVect)
{
cv::Mat newFrame = image.clone();
assert(!newFrame.empty());
std::vector<std::vector<int>> vecSeams;
for(int i = 0; i < seams; i++)
{
Log(Log::DEBUG)<<"Seam "<<i<<" of "<<seams;
//Gradient Magnitude for intensity of image.
cv::Mat gradientMagnitude = computeGradientMagnitude(image);
cv::Mat gradientMagnitude = computeGradientMagnitude(newFrame);
//Use DP to create the real energy map that is used for path calculation.
// Strictly using vertical paths for testing simplicity.
cv::Mat pathIntensityMat = computePathIntensityMat(gradientMagnitude);
if(pathIntensityMat.rows == 0 && pathIntensityMat.cols == 0)
return false;
std::vector<int> seam = getLeastImportantPath(pathIntensityMat);
vecSeams.push_back(seam);
if(seamsVect)
seamsVect->push_back(seam);
if(!grow)
image = removeLeastImportantPath(image, seam);
else
image = addLeastImportantPath(image, seam);
newFrame = removeLeastImportantPath(newFrame, seam);
if(image.rows == 0 && image.cols == 0)
if(newFrame.rows == 0 || newFrame.cols == 0)
return false;
}
if (grow)
{
cv::Mat growMat = image.clone();
for(size_t i = 0; i < vecSeams.size(); i++)
{
growMat = addLeastImportantPath(growMat,vecSeams[i]);
}
image = growMat;
}
else
{
image = newFrame;
}
return true;
}
bool SeamCarving::strechImageVert(cv::Mat& image, int seams, bool grow, std::vector<std::vector<int>>* seamsVect)
{
image = image.t();
cv::transpose(image, image);
bool ret = strechImage(image, seams, grow, seamsVect);
image = image.t();
cv::transpose(image, image);
return ret;
}