Unable to locate the lines using hough transform using opencv

Question

I want to find lines in the following image using hough transform and I am failing miserably. Could somebody tell where the problem is?

I am using the standard code from opencv.

I am using python and opencv 2.4.2

Findings:

1. these lines of rectangles are very jaggy
2. the edge detection finds broken edges
3. even you specify the parameter to connect to fill the gap it does not help.

Thanks a lot.

EDIT As suggested by "jpa" the image is inverted and the edge detection step is skipped as well

Here is the image after inversion

The parameter used are as follows

    HoughLinesP(image,10, math.pi/2 ,10       ,None ,1000,          1)
    HoughLinesP(image,rho, theta    ,threshold,lines,minLineLength, maxLineGap)

The output is the following where red color shows the presence of line.

Answer 1

Taking your original image as the input of the following program produces this as result:

The green lines represent what was successfully detected. The program is a slight modification of the original squares example that ships with OpenCV.

I's up to you to write the code that ignores the largest lines (which identify the paper).

The lines are stored in vector<vector<Point> > squares declared in main():

#include "highgui.h"
#include "cv.h"

#include <iostream>
#include <math.h>
#include <string.h>

using namespace cv;
using namespace std;

void help()
{
        cout <<
        "\nA program using pyramid scaling, Canny, contours, contour simpification and\n"
        "memory storage (it's got it all folks) to find\n"
        "squares in a list of images pic1-6.png\n"
        "Returns sequence of squares detected on the image.\n"
        "the sequence is stored in the specified memory storage\n"
        "Call:\n"
        "./squares\n"
    "Using OpenCV version %s\n" << CV_VERSION << "\n" << endl;
}


int thresh = 70, N = 2; 
const char* wndname = "Square Detection Demo";

// helper function:
// finds a cosine of angle between vectors
// from pt0->pt1 and from pt0->pt2
double angle( Point pt1, Point pt2, Point pt0 )
{
    double dx1 = pt1.x - pt0.x;
    double dy1 = pt1.y - pt0.y;
    double dx2 = pt2.x - pt0.x;
    double dy2 = pt2.y - pt0.y;
    return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}

// returns sequence of squares detected on the image.
// the sequence is stored in the specified memory storage
void findSquares( const Mat& image, vector<vector<Point> >& squares )
{
    squares.clear();

    Mat pyr, timg, gray0(image.size(), CV_8U), gray;

    // karlphillip: dilate the image so this technique can detect the white square,
    Mat out(image);
    dilate(out, out, Mat(), Point(-1,-1));
    // then blur it so that the ocean/sea become one big segment to avoid detecting them as 2 big squares.
    medianBlur(out, out, 3);

    // down-scale and upscale the image to filter out the noise
    pyrDown(out, pyr, Size(out.cols/2, out.rows/2));
    pyrUp(pyr, timg, out.size());
    vector<vector<Point> > contours;

    // find squares in every color plane of the image
    for( int c = 0; c < 1; c++ )
    {
        int ch[] = {c, 0};
        mixChannels(&timg, 1, &gray0, 1, ch, 1);

        // try several threshold levels
        for( int l = 0; l < N; l++ )
        {
            // hack: use Canny instead of zero threshold level.
            // Canny helps to catch squares with gradient shading
            if( l == 0 )
            {
                // apply Canny. Take the upper threshold from slider
                // and set the lower to 0 (which forces edges merging)
                Canny(gray0, gray, 0, thresh, 5);
                // dilate canny output to remove potential
                // holes between edge segments
                dilate(gray, gray, Mat(), Point(-1,-1));
            }
            else
            {
                // apply threshold if l!=0:
                //     tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
                gray = gray0 >= (l+1)*255/N;
            }

            // find contours and store them all as a list
            findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);

            vector<Point> approx;

            // test each contour
            for( size_t i = 0; i < contours.size(); i++ )
            {
                // approximate contour with accuracy proportional
                // to the contour perimeter
                approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);

                // square contours should have 4 vertices after approximation
                // relatively large area (to filter out noisy contours)
                // and be convex.
                // Note: absolute value of an area is used because
                // area may be positive or negative - in accordance with the
                // contour orientation
                if( approx.size() == 4 &&
                    fabs(contourArea(Mat(approx))) > 1000 &&
                    isContourConvex(Mat(approx)) )
                {
                    double maxCosine = 0;

                    for( int j = 2; j < 5; j++ )
                    {
                        // find the maximum cosine of the angle between joint edges
                        double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                        maxCosine = MAX(maxCosine, cosine);
                    }

                    // if cosines of all angles are small
                    // (all angles are ~90 degree) then write quandrange
                    // vertices to resultant sequence
                    if( maxCosine < 0.3 )
                        squares.push_back(approx);
                }
            }
        }
    }
}    

// the function draws all the squares in the image
void drawSquares( Mat& image, const vector<vector<Point> >& squares )
{
    for( size_t i = 1; i < squares.size(); i++ )
    {
        const Point* p = &squares[i][0];
        int n = (int)squares[i].size();
        polylines(image, &p, &n, 1, true, Scalar(0,255,0), 3, CV_AA);
    }

    imshow(wndname, image);
}    

int main(int argc, char** argv)
{
    if (argc < 2)
    {
        cout << "Usage: ./program <file>" << endl;
        return -1;
    }

    static const char* names[] = { argv[1], 0 };

    help();
    namedWindow( wndname, 1 );
    vector<vector<Point> > squares;

    for( int i = 0; names[i] != 0; i++ )
    {
        Mat image = imread(names[i], 1);
        if( image.empty() )
        {
            cout << "Couldn't load " << names[i] << endl;
            continue;
        }

        findSquares(image, squares);
        drawSquares(image, squares);
        imwrite("out.jpg", image);

        int c = waitKey();
        if( (char)c == 27 )
            break;
    }

    return 0;
}