iphone - adjust corners and crop the image openCV

Question

I am using open CV,in IOS. I already detected the boundary of the paper sheet in an Image as show in image and , Now I have to drag these boundary line on touch for adjusting the crop frame. how we can adjust boundary line and how we can crop image inside the boundary?

This is possible in openCV or I use openGL for this?

@moosgummi : I call your method in below method

- (cv::Mat)finshWork:(cv::Mat &)image
{

Mat img0 =image;

Mat img1;
cvtColor(img0, img1, CV_RGB2GRAY);

// apply your filter
Canny(img1, img1, 100, 200);

// find the contours
vector< vector<cv::Point> > contours;
findContours(img1, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);


// you could also reuse img1 here
Mat mask = Mat::zeros(img1.rows, img1.cols, CV_8UC1);

// CV_FILLED fills the connected components found
drawContours(mask, contours, -1, Scalar(255), CV_FILLED);


// let's create a new image now
Mat crop(img0.rows, img0.cols, CV_8UC3);

// set background to green
crop.setTo(Scalar(0,255,0));

// and copy the magic apple
img0.copyTo(crop, mask);

// normalize so imwrite(...)/imshow(...) shows the mask correctly!
normalize(mask.clone(), mask, 0.0, 255.0, CV_MINMAX, CV_8UC1);



std::vector<cv::Point> biggestContour = contours[contours.size()-1];

NSLog(@"%d",biggestContour[0].x);
NSLog(@"%d",biggestContour[0].y);

cv::Mat paperImage =[self getPaperAreaFromImage:image:biggestContour];


//return crop;
return paperImage;

}

Thanks All

Answer 1

After you got the corners you have to deskewing the paper and "extract" it to a new image.

You should do the following:

1. Sort corner points (the order matters; they must be in the same order in both vectors)
2. cv::getAffineTransform
3. cv::warpAffine

I wrote myself a helper function, which takes a std::vector with four cv::Point in it and sorts them in clockwise order beginning in the upper left. For more information on this topic take a look at these thread:

• Sort points in clockwise order?
• Sort Four Points in Clockwise Order
• Sorting of Points in 2D space

Another thing you should take into account is the size of the paper you want to extract. In my example I assume you're extracting a DIN A4 paper (210x297mm). Feel free to edit paperWidth and paperHeight inside my code.

Combining everything looks like this:

// Helper
cv::Point getCenter( std::vector<cv::Point> points ) {

    cv::Point center = cv::Point( 0.0, 0.0 );

    for( size_t i = 0; i < points.size(); i++ ) {
        center.x += points[ i ].x;
        center.y += points[ i ].y;
    }

    center.x = center.x / points.size();
    center.y = center.y / points.size();

    return center;

}

// Helper;
// 0----1
// |    |
// |    |
// 3----2
std::vector<cv::Point> sortSquarePointsClockwise( std::vector<cv::Point> square ) {

    cv::Point center = getCenter( square );

    std::vector<cv::Point> sorted_square;
    for( size_t i = 0; i < square.size(); i++ ) {
        if ( (square[i].x - center.x) < 0 && (square[i].y - center.y) < 0 ) {
            switch( i ) {
                case 0:
                    sorted_square = square;
                    break;
                case 1:
                    sorted_square.push_back( square[1] );
                    sorted_square.push_back( square[2] );
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    break;
                case 2:
                    sorted_square.push_back( square[2] );
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    sorted_square.push_back( square[1] );
                    break;
                case 3:
                    sorted_square.push_back( square[3] );
                    sorted_square.push_back( square[0] );
                    sorted_square.push_back( square[1] );
                    sorted_square.push_back( square[2] );
                    break;
            }
            break;
        }
    }

    return sorted_square;

}

// Helper
float distanceBetweenPoints( cv::Point p1, cv::Point p2 ) {

    if( p1.x == p2.x ) {
        return abs( p2.y - p1.y );
    }
    else if( p1.y == p2.y ) {
        return abs( p2.x - p1.x );
    }
    else {
        float dx = p2.x - p1.x;
        float dy = p2.y - p1.y;
        return sqrt( (dx*dx)+(dy*dy) );
    }
}

cv::Mat getPaperAreaFromImage( cv::Mat image, std::vector<cv::Point> square )
{

    // declare used vars
    int paperWidth  = 210; // in mm, because scale factor is taken into account
    int paperHeight = 297; // in mm, because scale factor is taken into account
    cv::Point2f imageVertices[4];
    float distanceP1P2;
    float distanceP1P3;
    BOOL isLandscape = true;
    int scaleFactor;
    cv::Mat paperImage;
    cv::Mat paperImageCorrected;
    cv::Point2f paperVertices[4];

    // sort square corners for further operations
    square = sortSquarePointsClockwise( square );

    // rearrange to get proper order for getPerspectiveTransform()
    imageVertices[0] = square[0];
    imageVertices[1] = square[1];
    imageVertices[2] = square[3];
    imageVertices[3] = square[2];

    // get distance between corner points for further operations
    distanceP1P2 = distanceBetweenPoints( imageVertices[0], imageVertices[1] );
    distanceP1P3 = distanceBetweenPoints( imageVertices[0], imageVertices[2] );

    // calc paper, paperVertices; take orientation into account
    if ( distanceP1P2 > distanceP1P3 ) {
        scaleFactor =  ceil( lroundf(distanceP1P2/paperHeight) ); // we always want to scale the image down to maintain the best quality possible
        paperImage = cv::Mat( paperWidth*scaleFactor, paperHeight*scaleFactor, CV_8UC3 );
        paperVertices[0] = cv::Point( 0, 0 );
        paperVertices[1] = cv::Point( paperHeight*scaleFactor, 0 );
        paperVertices[2] = cv::Point( 0, paperWidth*scaleFactor );
        paperVertices[3] = cv::Point( paperHeight*scaleFactor, paperWidth*scaleFactor );
    }
    else {
        isLandscape = false;
        scaleFactor =  ceil( lroundf(distanceP1P3/paperHeight) ); // we always want to scale the image down to maintain the best quality possible
        paperImage = cv::Mat( paperHeight*scaleFactor, paperWidth*scaleFactor, CV_8UC3 );
        paperVertices[0] = cv::Point( 0, 0 );
        paperVertices[1] = cv::Point( paperWidth*scaleFactor, 0 );
        paperVertices[2] = cv::Point( 0, paperHeight*scaleFactor );
        paperVertices[3] = cv::Point( paperWidth*scaleFactor, paperHeight*scaleFactor );
    }

    cv::Mat warpMatrix = getPerspectiveTransform( imageVertices, paperVertices );
    cv::warpPerspective(_image, paperImage, warpMatrix, paperImage.size(), cv::INTER_LINEAR, cv::BORDER_CONSTANT );

    // we want portrait output
    if ( isLandscape ) {
        cv::transpose(paperImage, paperImageCorrected);
        cv::flip(paperImageCorrected, paperImageCorrected, 1);
        return paperImageCorrected;
    }

    return paperImage;

}

Usage:

// ... get paper square ...

cv::Mat paperImage = getPaperAreaFromImage( srcImage, paperSquare );