微信【跳一跳】 opencv视觉识别 + 物理外挂

视频连接：http://v.youku.com/v_show/id_XMzMyNDQxNTA0OA==.html?spm=a2h3j.8428770.3416059.1

---

---

---

初入门C++ 与 opencv视觉库,写了一个跳一跳的物理挂，现在识别率还比较差，先记录下过程，以后在慢慢修改整理。

一、外挂结构

上位机：USB摄像头连接windows电脑，用作处理识别拍摄到图像数据。

下位机：STM32单片机，用于控制陀机附带电容笔进行物理点击。

单片机部分很简单，所以下文主要记录上位机的内容。

---

---

二、上位机程序框架

开发平台：Visual Studio 2012

思路：

1. 读取摄像头数据；
2. 提取图像中的手机屏幕、进行屏幕矫正；
3. 识别人物和方块并计算它们的距离（现方案识别部分采用模板匹配、模板使用photoshop预先裁剪好，加载入程序中）；
4. 把计算结果通过串口交给下位机，由下位机带动陀机进行物理点击；

---

---

三、程序源码

ScreenExtraction.cpp用于识别屏幕边缘、提取内容和矫正

#include "ScreenExtraction.h"
#include "main.h"

ScreenExtract::ScreenExtract()
{

}

ScreenExtract::ScreenExtract(Mat srcMat)
{
    setSrc(srcMat);
}

ScreenExtract::~ScreenExtract()
{
}
Mat ScreenExtract::getDst()
{
    return m_MatDstImageStand;
}

bool ScreenExtract::setSrc(Mat srcMat)
{
    m_MatSrcImage = srcMat.clone();
    m_MatEdgeImage = srcMat.clone();
    m_MatHoughImage = srcMat.clone();
    m_MatCornerImage = srcMat.clone();
    Mat temp(,,srcMat.type());
    m_MatDstImageLie = temp.clone();
    Mat temp2(,,srcMat.type());
    m_MatDstImageStand = temp2.clone();
    return true;
}

Mat ScreenExtract::runExtract()
{
    int cannyThrel=;
    int failCNT=;
    while()
    {
        m_MatEdgeImage = EdgeDection(m_MatEdgeImage,cannyThrel);
        m_MatHoughImage = HoughLine(m_MatEdgeImage);
        m_MatCornerImage = CornerHarris(m_MatHoughImage,m_PointPerspectiveSrcBuff);
        if(failCNT>)
        {
            throw("Extract fail");
            return m_MatSrcImage;
        }
        if(m_PointPerspectiveSrcBuff.size() == )
        {
            break;
        }
        else if(m_PointPerspectiveSrcBuff.size()>)
        {
            if(cannyThrel<)
            {
                cannyThrel+=;
            }
            failCNT++;
        }
        else if(m_PointPerspectiveSrcBuff.size()<)
        {
            if(cannyThrel>)
            {
                cannyThrel-=;
            }
            failCNT++;
        }
        waitKey();
    }

    m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff);
    m_MatDstImageStand = rotation(m_MatDstImageLie,);
    return m_MatDstImageStand;
}
Mat ScreenExtract::runFastExtract(Mat srcMat)
{
    m_MatSrcImage = srcMat.clone();
    m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff);
    m_MatDstImageStand = rotation(m_MatDstImageLie,);
    return m_MatDstImageStand;

}

//private
Mat ScreenExtract::EdgeDection(Mat srcImage,int cannyThrel)
{
    Mat cannyEdge;
    Mat dstImage = m_MatSrcImage.clone(); 

    //降噪
    blur(m_MatSrcImage,cannyEdge,Size(,));
#ifdef DEBUG_SHOW_ScreenExtract
    //namedWindow("blur()",CV_WINDOW_AUTOSIZE);
    //imshow(Windows_Edge,g_dstImage);
    imshow("blur()",cannyEdge);
#endif

    //运行Canny算子
    Canny(cannyEdge,cannyEdge,cannyThrel,cannyThrel*,);

    //先将g_dstImage内的所有元素设置为0
    dstImage = Scalar::all();

    //使用Canny算子输出的边缘图g_cannyDetectedEdges作为掩码，来将原图g_srcImage拷贝到目标图g_dstImage中
    srcImage.copyTo(dstImage,cannyEdge);

#ifdef DEBUG_SHOW_ScreenExtract
    //namedWindow("EdgeDection()",CV_WINDOW_AUTOSIZE);
    //imshow(Windows_Edge,g_dstImage);
    imshow("EdgeDection()",cannyEdge);
#endif

    return cannyEdge;
}

Mat ScreenExtract::HoughLine(Mat srcImage)
{
    Mat dstImage = srcImage.clone();
    //g_houghWithSrc = g_srcImage.clone();
    dstImage = Scalar::all();

    //houghlinesP
    //vector<Vec4i> mylines;
    //HoughLinesP(g_cannyDetectedEdges,mylines,1,CV_PI/180,valueA+1,valueB,valueC);
    ////循环遍历绘制每一条线段
    //  for( size_t i = 0; i < mylines.size(); i++ )
    //  {
    //      Vec4i lines = mylines[i];
    //    //line（）划线
    //      line( dstImage, Point(lines[0], lines[1]), Point(lines[2], lines[3]), Scalar(55,100,195), 1,CV_AA);// CV_AA);
    //  }  

    //houghlines
    vector<Vec2f> mylines;

    HoughLines(srcImage, mylines, , (CV_PI-0.2)/,srcImage.cols/, ,  );
    //HoughLines(g_cannyDetectedEdges, mylines, 1, CV_PI/180, valueA+1, valueB, valueC );  

    //依次在图中绘制出每条线段
    for( size_t i = ; i < mylines.size(); i++ )
    {
        float rho = mylines[i][], theta = mylines[i][];
        Point pt1, pt2;
        double a = cos(theta), b = sin(theta);
        double x0 = a*rho, y0 = b*rho;
        pt1.x = cvRound(x0 + *(-b));
        pt1.y = cvRound(y0 + *(a));
        pt2.x = cvRound(x0 - *(-b));
        pt2.y = cvRound(y0 - *(a));
        line( dstImage, pt1, pt2, Scalar(,,), , CV_AA);
        //line( g_houghWithSrc, pt1, pt2, Scalar(0,255,0), 1, CV_AA);
    }

#ifdef DEBUG_SHOW_ScreenExtract
    //显示图片
    //imshow(Windows_Hough,dstImage);
    //imshow(Windows_Hough,g_houghLine);
    imshow("HoughLine()",dstImage);
#endif

    return dstImage;
}

Mat ScreenExtract::CornerHarris( Mat srcImage ,vector<Point2f>  &vecPoint)
{
    //---------------------------【1】定义一些局部变量-----------------------------
    Mat g_srcImageClone = m_MatSrcImage.clone() ;
    Mat writeImage(m_MatSrcImage.rows,m_MatSrcImage.cols,m_MatSrcImage.type());
    Mat dstImage;//目标图
    Mat normImage;//归一化后的图
    Mat scaledImage;//线性变换后的八位无符号整型的图
    int iCornerThresh = ;

    //g_CornerWithHoughWithSrc = g_houghWithSrc;
    writeImage = Scalar::all();
    //---------------------------【2】初始化---------------------------------------
    //置零当前需要显示的两幅图，即清除上一次调用此函数时他们的值
    //dstImage = Mat::zeros( srcImage.size(), CV_32FC1 );
    //g_srcImageClone=g_srcImage.clone( );  

    //---------------------------【3】正式检测-------------------------------------
    //进行角点检测
    //cornerHarris( g_srcGrayImage, dstImage, 2, 3, 0.04, BORDER_DEFAULT );  //原图角点检测
    cornerHarris( srcImage, dstImage, , , 0.04, BORDER_DEFAULT );    //霍夫变换后的角点检测

    // 归一化与转换
    normalize( dstImage, normImage, , , NORM_MINMAX, CV_32FC1, Mat() );
    convertScaleAbs( normImage, scaledImage );//将归一化后的图线性变换成8位无符号整型   

    //---------------------------【4】进行绘制-------------------------------------
    // 将检测到的，且符合阈值条件的角点绘制出来
    for( int j = normImage.rows/; j < normImage.rows-normImage.rows/ ; j++ )
    {
        for( int i = normImage.cols/; i < normImage.cols-normImage.cols/; i++ )
        {
            if( (int) normImage.at<float>(j,i) > iCornerThresh  )
            {
                circle( g_srcImageClone, Point( i, j ), ,  Scalar(,,), , ,  );
                //circle( g_CornerWithHoughWithSrc, Point( i, j ), 5,  Scalar(10,10,255), 2, 8, 0 );
                circle( scaledImage, Point( i, j ), ,  Scalar(,,), -, ,  );
                circle( writeImage, Point( i, j ), ,  Scalar(,,), -, ,  );
            }
        }
    }  

    vecPoint = GatherPoint( writeImage,iCornerThresh);
#ifdef DEBUG_SHOW_ScreenExtract
    //---------------------------【4】显示最终效果---------------------------------
    //imshow( "CornerHarris", g_CornerWithHoughWithSrc );  //在原图叠加霍夫图上显示
    //imshow( "CornerHarris", g_srcImageClone );  //在原图上显示
    //imshow( "CornerHarris", scaledImage );  //使用灰度图显示
    imshow( "CornerHarris", writeImage );
#endif
    return scaledImage;
}  

vector<Point2f> ScreenExtract::GatherPoint( Mat srcImage, int CornerThresh )
{
    //imshow( "GatherPoint", srcImage );
    //Mat grayImage=srcImage.clone() ;
    Mat canny_output;
    Mat grayImage;
    vector<vector<Point>>contours;
    vector<Vec4i>hierarchy;
    //RNG rng(12345);

    //转成灰度图
    cvtColor(srcImage, grayImage, COLOR_BGR2GRAY);

    //canny边缘检测
    Canny(grayImage, canny_output, ,  * , );

    //轮廓提取
    findContours(canny_output, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE, Point(, ));

    //计算图像矩
    vector<Moments>mu(contours.size());
    for (unsigned int i = ; i < contours.size(); i++)
    {
        mu[i] = moments(contours[i], false);
    }
    //计算图像的质心
    vector<Point2f>mc(contours.size());
    for (unsigned int i = ; i < contours.size(); i++)
    {
        mc[i] = Point2f(mu[i].m10 / mu[i].m00, mu[i].m01 / mu[i].m00);
    }

    //绘制轮廓
    Mat drawing = Mat::zeros(srcImage.size(), CV_8UC3);
    for (unsigned int i = ; i < contours.size(); i++)
    {
        Scalar color = Scalar(, , );
        drawContours(drawing, contours, i, color, , , hierarchy, , Point());
        circle(drawing, mc[i], , color, -, , );
    }
#ifdef DEBUG_SHOW_ScreenExtract
    //namedWindow("GatherPoint", WINDOW_AUTOSIZE);
    imshow("GatherPoint", drawing);
#endif

    return mc;
}

Mat ScreenExtract::perspectiveChange(Mat srcMat,vector<Point2f> srcBuff)
{
    //Mat变量
    Mat dstMat(srcMat.rows,srcMat.cols,srcMat.type());
    Mat perspectiveMat( ,,CV_32FC1);

    //透视变换参数
    Point2f perspectiveSrcBuff[];
    Point2f perspectiveDesBuff[];

    //透视变换坐标设置
    perspectiveSrcBuff[] = Point2f((srcBuff[].x+srcBuff[].x)/,(srcBuff[].y+srcBuff[].y)/) ;
    perspectiveSrcBuff[] = Point2f((srcBuff[].x+srcBuff[].x)/,(srcBuff[].y+srcBuff[].y)/) ;
    perspectiveSrcBuff[] = Point2f((srcBuff[].x+srcBuff[].x)/,(srcBuff[].y+srcBuff[].y)/) ;
    perspectiveSrcBuff[] = Point2f((srcBuff[].x+srcBuff[].x)/,(srcBuff[].y+srcBuff[].y)/) ;

    //求变换后坐标
    for(int i=;i<;i++)
    {
        if(== getPointPlace(srcMat,perspectiveSrcBuff[i]) )
        {
            perspectiveDesBuff[i] = Point2f( , );
        }
        else if( == getPointPlace(srcMat,perspectiveSrcBuff[i]) )
        {
            perspectiveDesBuff[i] = Point2f( , static_cast<float>(dstMat.rows-));
        }
        else if( == getPointPlace(srcMat,perspectiveSrcBuff[i]) )
        {
            perspectiveDesBuff[i] = Point2f( static_cast<float>(dstMat.cols-), );
        }
        else if( == getPointPlace(srcMat,perspectiveSrcBuff[i]) )
        {
            perspectiveDesBuff[i] = Point2f( static_cast<float>(dstMat.cols-), static_cast<float>(dstMat.rows-));
        }
    }

    //求透视变换
    perspectiveMat = getPerspectiveTransform( perspectiveSrcBuff, perspectiveDesBuff );  

    //对源图像应用刚刚的透视变换
    warpPerspective(srcMat, dstMat, perspectiveMat, dstMat.size());

#ifdef DEBUG_SHOW_ScreenExtract
    //显示
    imshow( "perspectiveChange", dstMat );
#endif
    return dstMat;
}
int ScreenExtract::getPointPlace(Mat srcImage,Point2f point)
{
    if(point.x < srcImage.cols/)
    {
        if(point.y < srcImage.rows/)
        {
            return ;
        }
        else
        {
            return ;
        }
    }
    else
    {
        if(point.y < srcImage.rows/)
        {
            return ;
        }
        else
        {
            return ;
        }
    }
}

Mat ScreenExtract::rotation(Mat srcMat,float degree)
{
    Mat dstImage(srcMat.rows,srcMat.cols,srcMat.type());
    int len = max(srcMat.cols, srcMat.rows);
    Point2f pt(len/.f,len/.f);
    Mat r = getRotationMatrix2D(pt,degree,1.0);
    warpAffine(srcMat,dstImage,r,Size(srcMat.rows,srcMat.cols));

    return dstImage;
}

ScreenExtraction.h为头文件

#ifndef __SCREENEXTRACTION_H
#define __SCREENEXTRACTION_H

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
/*
#include <opencv2/nonfree/nonfree.hpp>
#include<opencv2/legacy/legacy.hpp> */
#include <iostream>
using namespace cv;
using namespace std;

//#define DEBUG_SHOW_ScreenExtract

class ScreenExtract
{
public:
    ScreenExtract();
    ScreenExtract(Mat srcMat);
    ~ScreenExtract();
    Mat getDst();

    bool setSrc(Mat srcMat);
    Mat runExtract();
    Mat runFastExtract(Mat srcMat);
private:
    Mat EdgeDection(Mat srcImage,int cannyThrel);
    Mat HoughLine(Mat srcImage);
    Mat CornerHarris( Mat srcImage,vector<Point2f> &vecPoint) ;
    vector<Point2f> GatherPoint( Mat srcImage, int CornerThresh ) ;
    Mat perspectiveChange(Mat srcMat,vector<Point2f> srcBuff);
    int ScreenExtract::getPointPlace(Mat srcImage,Point2f point);//获取点在屏幕的位置
    Mat rotation(Mat srcImage,float degree);
private:
    Mat m_MatSrcImage;
    Mat m_MatEdgeImage;
    Mat m_MatHoughImage;
    Mat m_MatCornerImage;
    Mat m_MatDstImageLie;
    Mat m_MatDstImageStand;
    vector<Point2f> m_PointPerspectiveSrcBuff;
};

#endif

ImageMatch.cpp用于识别内容

#include "ImageMatch.h"

ImageMatch::ImageMatch(Mat srcImage)
{
    m_MatSrcImage = srcImage.clone();
    m_MatMatchImage = srcImage.clone();
    m_MatDstImage = srcImage.clone();
    m_iFeatureMaxSize = ;
}

ImageMatch::~ImageMatch()
{    

}
/*******************************************************************************
* Function Name  : addObjectFeatureImage
* Description    : add an objcet feature to feature vector
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool ImageMatch::setSrcImage(Mat srcImage)
{
    m_MatSrcImage = srcImage.clone();
    return true;
}
/*******************************************************************************
* Function Name  : addObjectFeatureImage
* Description    : add an objcet feature to feature vector
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool ImageMatch::addCubeFeatureImage(Mat image)
{
    if(m_vecCubeFeatureImage.size()<m_iFeatureMaxSize)
    {
        m_vecCubeFeatureImage.push_back(image);
        return true;
    }

    return false;
}
bool ImageMatch::addCubeFeatureImage(vector<Mat> vec)
{

    if(m_vecCubeFeatureImage.size()+vec.size()<m_iFeatureMaxSize)
    {
        m_vecCubeFeatureImage.insert(m_vecCubeFeatureImage.end(),vec.begin(),vec.end());
        return true;
    }

    return false;
}

bool ImageMatch::addConfusingFeatureImage(Mat image)
{
    if(m_vecConfusingFeatureImage.size()<m_iFeatureMaxSize)
    {
        m_vecConfusingFeatureImage.push_back(image);
        return true;
    }
    return false;
}
bool ImageMatch::addConfusingFeatureImage(vector<Mat> vec)
{

    if(m_vecConfusingFeatureImage.size()+vec.size()<m_iFeatureMaxSize)
    {
        m_vecConfusingFeatureImage.insert(m_vecConfusingFeatureImage.end(),vec.begin(),vec.end());
        return true;
    }

    return false;
}
/*******************************************************************************
* Function Name  : setPersonFeature
* Description    : set the person feature
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool ImageMatch::setPersonFeature(Mat image)
{
    m_MatPersonFeatureImage = image.clone();
    return true;
}

/*******************************************************************************
* Function Name  : runMatch
* Description    : Match main programm
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
Mat ImageMatch::runMatch()
{
    //【0】预处理
    Mat dstImage = m_MatSrcImage.clone();
    Mat srcImage_Adjust= AdjectDefinition(m_MatSrcImage,,);

    //【1】匹配人物
    m_pLocPerson = SearchPerson( srcImage_Adjust, , (int)(srcImage_Adjust.rows*0.35), srcImage_Adjust.cols-, (int)(srcImage_Adjust.rows*0.35) );
#ifdef DEBUG_SHOW_ImageMatch
    //rectangle( srcImage_Adjust,  Rect(loc,  Size(m_MatPersonFeatureImage.cols, m_MatPersonFeatureImage.rows) ), Scalar(0, 0, 255), 2, 8, 0 );
    circle( srcImage_Adjust, m_pLocPerson, ,  Scalar(,,), , ,  );
    //imshow("TempleMatch",srcImage_Adjust);
#endif

    //【2】匹配建筑
    if(m_pLocPerson.x < srcImage_Adjust.cols*0.5)//人物在左边，搜索右半区域，为了加速
        m_pLocBlock = SearchBuilding( srcImage_Adjust, (int)(srcImage_Adjust.cols*0.3)+, (int)(srcImage_Adjust.rows*0.20), (int)(srcImage_Adjust.cols*0.7)-, (int)(srcImage_Adjust.rows*0.5));
    else//搜索左半区
        m_pLocBlock = SearchBuilding( srcImage_Adjust , , (int)(srcImage_Adjust.rows*0.20), (int)(srcImage_Adjust.cols*0.7)-, (int)(srcImage_Adjust.rows*0.5));
#ifdef DEBUG_SHOW_ImageMatch
    circle( srcImage_Adjust, m_pLocBlock, ,  Scalar(,,), , ,  );  

    imshow("TempleMatch",srcImage_Adjust);
#endif

    m_fDistance    = calculateDistance(m_pLocPerson,m_pLocBlock);

    return dstImage;
}
/*******************************************************************************
* Function Name  : changeSrcAndrunMatch
* Description    : Change src image and run Match
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
Mat ImageMatch::changeSrcAndrunMatch(Mat srcImage)
{
    setSrcImage(srcImage);
    return runMatch();
}
float ImageMatch::getDistance()
{
    float result = m_fDistance*2.1f+;
    if(result<)
        result=;
    return result;
}
//private
/*******************************************************************************
* Function Name  : vecFeatureImageToVecFeatureImage
* Description    : update definition
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
bool ImageMatch::FeatureVectorChangeToEdgeVector()
{
    m_vecFeatureEdgeImage.clear();
    for(unsigned int i=;i<m_vecCubeFeatureImage.size();i++)
    {
        //m_MatSrcImage = FeatureMatchAndMark(m_vecCubeFeatureImage[i],m_MatSrcImage);
        m_vecFeatureEdgeImage.push_back ( calEdge(m_vecCubeFeatureImage[i],));
#ifdef DEBUG_SHOW_ImageMatch
        imshow("Edge",m_vecFeatureEdgeImage[i]);
#endif
    }
    return true;
}
/*******************************************************************************
* Function Name  : definition
* Description    : update definition
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
Mat ImageMatch::AdjectDefinition(Mat srcImage,int contrast,int brightness)
{
    Mat dstImage;
    //平滑
    blur(srcImage,dstImage,Size(,));

    //对比度亮度调节
    for(int y = ; y < dstImage.rows; y++ ) //遍历图片的纵坐标
    {
        for(int x = ; x < dstImage.cols; x++ )//遍历图片的横坐标
        {
            for(int c = ; c < ; c++ )  //分开图像的RGB
            {
                //对比度在0-300之间，所以乘以0.01,并用saturate_cast把计算结果转换成uchar类型
                dstImage.at<Vec3b>(y,x)[c]= saturate_cast<uchar>( (contrast*0.01)*(dstImage.at<Vec3b>(y,x)[c] ) + brightness );
            }
        }
    }
    //    imshow("src",srcImage);
    //    imshow("dst",dstImage);

    //二次平滑
    //blur(dstImage,dstImage,Size(3,3)); 

    return dstImage;
}
/*******************************************************************************
* Function Name  : addObjectFeatureImage
* Description    : add an objcet feature to feature vector
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
Mat ImageMatch::calEdge(Mat srcImage,int threl)
{
    Mat dstImage;
    //canny
    Canny(srcImage,dstImage,threl,threl*,);

#ifdef DEBUG_SHOW_ImageMatch
    imshow("calEdge",dstImage);

#endif
    return dstImage;
}

/*******************************************************************************
* Function Name  : SearchPerson
* Description    : find block point
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
Point ImageMatch::SearchPerson(Mat TargetImage,int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh )
{
    //【0】局部变量用于返回人物坐标
    Point dstLoc;

    //【1】设置ROI区域
    Mat imageROI=TargetImage( Rect(ROI_Xstart, ROI_Ystart, ROI_length, ROI_heigh ) );    

    //【2】模板匹配
    float matchaValue = (float)TempleMatch(m_MatPersonFeatureImage, imageROI, dstLoc, CV_TM_CCOEFF_NORMED) ;
    if( matchaValue < 0.64)
    {
        throw("Person match fail");
    }
    else
    {
#ifdef DEBUG_SHOW_ImageMatch
        //rectangle( imageROI,  Rect(dstLoc,  Size(m_MatPersonFeatureImage.cols, m_MatPersonFeatureImage.rows) ), Scalar(0, 0, 255), 2, 8, 0 );
        //imshow("SearchqPerson",imageROI);
#endif
        //根据输入的ROI设置纠正坐标
        dstLoc.x+=ROI_Xstart;
        dstLoc.y+=ROI_Ystart;
        //让坐标重新纠正道指向人物与地面的接触中点
        dstLoc.x += (int)(m_MatPersonFeatureImage.cols/);
        dstLoc.y += (int)(m_MatPersonFeatureImage.rows*0.8);
    }
    return dstLoc;
}

/*******************************************************************************
* Function Name  : FindBuilding
* Description    : find block point
* Input          : None
* Output         : None
* Return         : destination image
*******************************************************************************/
Point ImageMatch::SearchBuilding(Mat TargetImage, int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh)
{
    //【0】定义两个容器用于存放每个矩形特征的最佳匹配点和匹配值
    vector<Point> vecCubePoint(m_vecCubeFeatureImage.size());
    vector<float> vecCubeValue(m_vecCubeFeatureImage.size());
    //【0】定义一些阈值
    float completelyMatchThrel = 0.70f;//认为一种图形完全匹配的阈值
    float LowestThrel = ;//认为基本相近的阈值
    //【0】定义一些临时变量
    Point loc;
    float matchValue;
    //【0】最终配置相关变量
    bool matchSucceed=false;//是否找到完全匹配特征标记
    Point lastLoc(,);//最终位置
    float lastValue;//最终匹配值
    int lastIndex=-;//最终位置对应的特征序号

    //【0】计时
    TimeOperation time;
    int ms=;

    //【1】转为灰度图
    cvtColor(TargetImage,TargetImage,CV_BGR2GRAY);

    //【2】设置ROI区域
    Mat imageROI=TargetImage( Rect(ROI_Xstart, ROI_Ystart, ROI_length, ROI_heigh ) );    

    //【3】匹配建筑
    for(unsigned int i=;i<m_vecCubeFeatureImage.size();i++)
    {
        matchValue = (float)TempleMatch(m_vecCubeFeatureImage[i],imageROI,loc,CV_TM_CCOEFF_NORMED );
        if(matchValue > completelyMatchThrel)
        {
            lastLoc = loc ;
            //纠正坐标
            lastLoc.x += ROI_Xstart;
            lastLoc.y += ROI_Ystart;
            lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*0.5);
            lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3);
            //若是任务脚底的建筑，继续找
            if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel+)
            {
                lastLoc.x=;
                lastLoc.y=;
                continue;
            }
            matchSucceed = true;
            lastValue = matchValue;
            lastIndex = i;
#ifdef DEBUG_SHOW_ImageMatch
            imshow("matchfeature",m_vecCubeFeatureImage[i]);
            rectangle( imageROI,  Rect(loc,  Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(, , ), , ,  );
            imshow("SearchPerson",imageROI);
#endif
            break;
        }
        else
        {
            vecCubePoint[i]=loc;
            vecCubeValue[i]=matchValue;
        }
    }

    ////【4】没有完全匹配，寻找最佳匹配
    if(!matchSucceed)
    {
        for(unsigned int i=;i<m_vecCubeFeatureImage.size();i++)
        {
            Mat featureROI;
            if(m_pLocPerson.x<TargetImage.cols/)//取特征的右边
            {
                featureROI =m_vecCubeFeatureImage[i]( Rect(m_vecCubeFeatureImage[i].cols*0.4, , m_vecCubeFeatureImage[i].cols*0.6-, m_vecCubeFeatureImage[i].rows ) );
                matchValue = (float)TempleMatch(featureROI,imageROI,loc,CV_TM_CCOEFF_NORMED );
                if(matchValue > completelyMatchThrel)
                {
                    lastLoc = loc ;
                    //纠正坐标
                    lastLoc.x += ROI_Xstart;
                    lastLoc.y += ROI_Ystart;
                    lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*(0.5-0.4));
                    lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3);
                    //若是任务脚底的建筑，继续找
                    if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel)
                    {
                        lastLoc.x=;
                        lastLoc.y=;
                        continue;
                    }
                    matchSucceed = true;
                    lastValue = matchValue;
                    lastIndex = i;
#ifdef DEBUG_SHOW_ImageMatch
                    imshow("matchfeature",m_vecCubeFeatureImage[i]);
                    rectangle( imageROI,  Rect(loc,  Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(, , ), , ,  );
                    imshow("SearchPerson",imageROI);
#endif
                    break;
                }
            }
            else
            {
                featureROI =m_vecCubeFeatureImage[i]( Rect(, , m_vecCubeFeatureImage[i].cols*0.6-, m_vecCubeFeatureImage[i].rows ) );
                matchValue = (float)TempleMatch(featureROI,imageROI,loc,CV_TM_CCOEFF_NORMED );
                if(matchValue > completelyMatchThrel)
                {
                    lastLoc = loc ;
                    //纠正坐标
                    lastLoc.x += ROI_Xstart;
                    lastLoc.y += ROI_Ystart;
                    lastLoc.x += (int)(m_vecCubeFeatureImage[i].cols*0.5);
                    lastLoc.y += (int)(m_vecCubeFeatureImage[i].rows*0.3);
                    //若是任务脚底的建筑，继续找
                    if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel)
                    {
                        lastLoc.x=;
                        lastLoc.y=;
                        continue;
                    }
                    matchSucceed = true;
                    lastValue = matchValue;
                    lastIndex = i;
#ifdef DEBUG_SHOW_ImageMatch
                    imshow("matchfeature",m_vecCubeFeatureImage[i]);
                    rectangle( imageROI,  Rect(loc,  Size(m_vecCubeFeatureImage[i].cols, m_vecCubeFeatureImage[i].rows) ), Scalar(, , ), , ,  );
                    imshow("SearchPerson",imageROI);
#endif
                    break;
                }
            }

        }
    }
    //易混淆图片匹配
    for(unsigned int i=;i<m_vecConfusingFeatureImage.size();i++)
    {
        matchValue = (float)TempleMatch(m_vecConfusingFeatureImage[i],imageROI,loc,CV_TM_CCOEFF_NORMED );
        if(matchValue > 0.95)
        {
            lastLoc = loc ;
            //纠正坐标
            lastLoc.x += ROI_Xstart;
            lastLoc.y += ROI_Ystart;
            lastLoc.x += (int)(m_vecConfusingFeatureImage[i].cols*0.5);
            lastLoc.y += (int)(m_vecConfusingFeatureImage[i].rows*0.3);
            //若是任务脚底的建筑，继续找
            if(calculateDistance(lastLoc,m_pLocPerson)<LowestThrel+)
            {
                lastLoc.x=;
                lastLoc.y=;
                continue;
            }
            matchSucceed = true;
            lastValue = matchValue;
            lastIndex = i;
#ifdef DEBUG_SHOW_ImageMatch
            imshow("matchfeature",m_vecConfusingFeatureImage[i]);
            rectangle( imageROI,  Rect(loc,  Size(m_vecConfusingFeatureImage[i].cols, m_vecConfusingFeatureImage[i].rows) ), Scalar(, , ), , ,  );
            imshow("SearchPerson",imageROI);
#endif
            break;
        }
        else
        {
            vecCubePoint[i]=loc;
            vecCubeValue[i]=matchValue;
        }
    }

    if(!matchSucceed)
    {
        float bestValue=;
        int bestIndex = ;
        unsigned int i=;

        bestValue = vecCubeValue[];
        for(i=;i<vecCubeValue.size();i++)
        {
            if(vecCubeValue[i]>bestValue)
            {
                bestValue = vecCubeValue[i];
                bestIndex = i;
            }
        }

#ifdef DEBUG_SHOW_ImageMatch
        imshow("matchfeature",m_vecCubeFeatureImage[bestIndex]);
#endif
        lastLoc = vecCubePoint[bestIndex];
        lastIndex=bestIndex;
    }
    return lastLoc;
}
/*******************************************************************************
* Function Name  : calculateDistance
* Description    : Temple match ,Get the best matching point and return it
* Input          : None
* Output         : None
* Return         : Point:best matching point.
*******************************************************************************/
float ImageMatch::calculateDistance(Point A,Point B)
{
    float distance;
    distance = powf((float)(B.x - A.x),) + powf((float)(B.y - A.y),);
    distance = sqrtf(distance);
    return distance;
}
/*******************************************************************************
* Function Name  : TempleMatch
* Description    : Temple match ,Get the best matching point and return it
* Input          : None
* Output         : None
* Return         : Point:best matching point.
*******************************************************************************/
double ImageMatch::TempleMatch( Mat tepl ,Mat image, Point &point,int method)
{
    int result_cols =  image.cols - tepl.cols + ;
    int result_rows = image.rows - tepl.rows + ;

    //模板检测
    Mat result =  Mat( result_cols, result_rows, CV_32FC1 );
    matchTemplate( image, tepl, result, method );

    //寻找最佳值
    double minVal, maxVal;
    Point minLoc, maxLoc;
    minMaxLoc( result, &minVal, &maxVal, &minLoc, &maxLoc, Mat() );

    //根据方法返回最佳值
    if(CV_TM_SQDIFF == method || CV_TM_SQDIFF_NORMED == method)
    {
        point =  minLoc;
        return minVal;
    }
    else
    {
        point = maxLoc;
        return maxVal;
    }
    return ;
}

ImageMatch.h为头文件

#ifndef __IMAGEMATCH_H
#define __IMAGEMATCH_H

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/core/core.hpp>  

#include <iostream>
#include <vector>
#include "TimeOperation.h"

#define DEBUG_SHOW_ImageMatch

using namespace std;
using namespace cv;

class ImageMatch
{
public:
    ImageMatch(Mat srcImage);
    ~ImageMatch();
    bool setSrcImage(Mat srcImage);
    bool addCubeFeatureImage(Mat image);
    bool addCubeFeatureImage(vector<Mat> vec);
    bool addConfusingFeatureImage(Mat image);
    bool addConfusingFeatureImage(vector<Mat> vec);
    bool setPersonFeature(Mat image);

    Mat runMatch();
    Mat changeSrcAndrunMatch(Mat srcImage);
    float getDistance();
private:
    bool FeatureVectorChangeToEdgeVector();
    Mat AdjectDefinition(Mat srcImage,int contrast=,int brightness=);
    Mat calEdge(Mat srcImage,int threl);
    Mat FeatureMatchAndMark(Mat FeatureImage,Mat TargetImage);
    Point SearchPerson(Mat TargetImage,int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh );
    Point SearchBuilding(Mat TargetImage, int ROI_Xstart,int ROI_Ystart,int ROI_length,int ROI_heigh);
    float calculateDistance(Point A,Point B);
private:
    double TempleMatch( Mat tepl ,Mat image, Point &point,int method);
private:
    vector<Mat> m_vecCubeFeatureImage;
    vector<Mat> m_vecConfusingFeatureImage;
    vector<Mat> m_vecFeatureEdgeImage;
    unsigned int m_iFeatureMaxSize;
    Mat m_MatPersonFeatureImage;
    Mat m_MatSrcImage;
    Mat m_MatMatchImage;
    Mat m_MatDstImage;
    float m_fDistance;
    Point m_pLocPerson;
    Point m_pLocBlock;
};

#endif

SeralPort.cpp为串口相关内容，用于与下位机通信

#include "SerialPort.h"

/*******************************************************************************
* Function Name  : SerialPort
* Description    : open serial port.
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
SerialPort::SerialPort(int CommNumber,int BaudRate,int Parity ,int ByteSize , int StopBits )
{
    //【0】预处理
    if(CommNumber< || BaudRate< || Parity< || ByteSize< || StopBits<)
    {
        throw("SerialPort parameter wrong");
        return;
    }
    m_iComNumber = CommNumber;

    //int 2 str
    std::stringstream ss;
    std::string str;
    ss<<CommNumber;
    ss>>str;

    string strComNumber = "COM" + str;
    const  char *pComNumber =  strComNumber.c_str();

    //【1】打开串口
    m_cHCom = CreateFileA( pComNumber, GENERIC_READ | GENERIC_WRITE, , NULL, OPEN_EXISTING, , );
    //【1.5】超时处理
    COMMTIMEOUTS stTimeOuts;
    GetCommTimeouts(m_cHCom, &stTimeOuts);
    stTimeOuts.ReadIntervalTimeout = ;
    stTimeOuts.ReadTotalTimeoutMultiplier = ;
    stTimeOuts.ReadTotalTimeoutConstant = ;
    stTimeOuts.WriteTotalTimeoutMultiplier = ;
    stTimeOuts.WriteTotalTimeoutConstant = ;
    if (!SetCommTimeouts(m_cHCom, &stTimeOuts))
    {
        throw("SerialPort openning time out");
        CloseHandle(m_cHCom);
        return;
    }

    //【2】取得并设置端口状态
    GetCommState(m_cHCom, &m_cDcb);
    m_cDcb.DCBlength = sizeof(DCB);
    m_cDcb.BaudRate = BaudRate;
    m_cDcb.Parity = Parity;
    m_cDcb.ByteSize = ByteSize;
    m_cDcb.StopBits = StopBits;
    if (!SetCommState(m_cHCom, &m_cDcb))
    {
        throw("SerialPort openning fail");
        CloseHandle(m_cHCom);
        return;
    }
}

SerialPort::~SerialPort( )
{
    CloseHandle(m_cHCom);
}

/*******************************************************************************
* Function Name  : SerialPort::send
* Description    : serialPort send.
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool SerialPort::send(vector<unsigned char> vecSend)
{
    DWORD dwWrittenLen;
    //【0】 把vector换成char*
    char *cSendBuff = new char[vecSend.size()];
    {
         memcpy(cSendBuff, &vecSend[], vecSend.size()*sizeof(char));
    }

    if(!WriteFile(m_cHCom,cSendBuff, vecSend.size(),&dwWrittenLen,NULL))
    {
        throw("Sending fail 1");
        return false;
    }
    if(vecSend.size() != dwWrittenLen)
    {
        throw("Sending fail 2");
        return false;
    }
    return true;
}

/*******************************************************************************
* Function Name  : SerialPort::send
* Description    : serialPort send.
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool SerialPort::packAndsend(unsigned short data1)
{
    const int PackageLength=;
    const unsigned char PackHead1=0x55;
    const unsigned char PackHead2=0xaa;

    static unsigned char number = ;

    unsigned short check=;
    vector<unsigned char> vecForSend(PackageLength);

    //报头
    vecForSend[] = PackHead1;
    vecForSend[] = PackHead2;
    //序号
    vecForSend[] = number++;
    //时间
    vecForSend[] = (char)(data1>>);
    vecForSend[] = (char)data1;

    //校验
    for(int i=;i<PackageLength-;i++)
    {
        check += vecForSend[i];
    }
    vecForSend[] = (char)(check>>);
    vecForSend[] = (char)check;

    //发送
    if(send(vecForSend))
    {
        return true;
    }

    return false;
}

SeralPort.h为头文件

#ifndef __SERIALPORT_H
#define __SERIALPORT_H

#include <windows.h>
#include <iostream>
#include <vector>
#include <string>
#include <sstream>

using namespace std;

class SerialPort
{
public:
    SerialPort(int CommNumber,int BaudRate = ,int Parity = ,int ByteSize = , int StopBits = ONESTOPBIT);
    ~SerialPort();
public:
    bool send(vector<unsigned char> vecSend);
    bool packAndsend(unsigned short data1);
private:
    HANDLE m_cHCom;
    int m_iComNumber;
    DCB m_cDcb;
    vector<char> m_vecRecived;
};

#endif

TimeOperation.cpp用于计时、定时等操作

#include "TimeOperation.h"

using namespace std;

TimeOperation::TimeOperation()
{
    m_eSingleStopwatchState = STOPWATCH_STOP;
    m_eTimeOutDectectionState = TIMEOUTCLOCK_OFF;
}

TimeOperation::~TimeOperation()
{

}

/*******************************************************************************
* Function Name  : singleStopwatchRestart
* Description    : restart stopwatch
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::singularStopwatchRestart()
{
    m_eSingleStopwatchState = STOPWATCH_ON;
    GetLocalTime(&m_cStopwatchOrigin);

    //cout<<"start.wSecond:"<<m_cStopwatchOrigin.wMinute<<endl;
    //cout<<"start.wMilliseconds:"<<m_cStopwatchOrigin.wMilliseconds<<endl;

    return true;
}
/*******************************************************************************
* Function Name  : singleStopwatchPause
* Description    : pause stopwatch and get time (In Milliseconds).
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::singularStopwatchPause(int &ms)
{
    if(STOPWATCH_ON != m_eSingleStopwatchState )
    {
        return false;
    }

    SYSTEMTIME EndTime;
    GetLocalTime(&EndTime);

    //cout<<"end.wSecond:"<<EndTime.wMinute<<endl;
    //cout<<"end.wMilliseconds:"<<EndTime.wMilliseconds<<endl;

    ms = EndTime.wMilliseconds-m_cStopwatchOrigin.wMilliseconds + ((EndTime.wSecond-m_cStopwatchOrigin.wSecond) + (EndTime.wMinute - m_cStopwatchOrigin.wMinute)*)*;

    cout<<"ms:"<<ms<<endl;

    m_eSingleStopwatchState = STOPWATCH_PAUSE;

    return true;
}
/*******************************************************************************
* Function Name  : singleStopwatchPause
* Description    : pause stopwatch and get time (In Milliseconds).
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::TimeOutDectectionSetClock(SYSTEMTIME &setTime)
{
    m_cTimeOutDectectionThrel = ((setTime.wHour*+setTime.wMinute)*+setTime.wSecond)*+setTime.wMilliseconds;

    //set now time
    SYSTEMTIME nowTime;
    GetLocalTime(&nowTime);
    m_cTimeOutDectectionStartTime = ((nowTime.wHour*+nowTime.wMinute)*+nowTime.wSecond)*+nowTime.wMilliseconds;

    m_eTimeOutDectectionState = TIMEOUTCLOCK_ON;
    return true;
}
/*******************************************************************************
* Function Name  : TimeOutDectectionSetClock
* Description    : pause stopwatch and get time (In Milliseconds).
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::TimeOutDectectionSetClock(int ms)
{
    m_cTimeOutDectectionThrel = ms;

    SYSTEMTIME nowTime;
    GetLocalTime(&nowTime);
    m_cTimeOutDectectionStartTime = ((nowTime.wHour*+nowTime.wMinute)*+nowTime.wSecond)*+nowTime.wMilliseconds;

    m_eTimeOutDectectionState = TIMEOUTCLOCK_ON;
    return true;
}
/*******************************************************************************
* Function Name  : singleStopwatchPause
* Description    : pause stopwatch and get time (In Milliseconds).
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::singularTimeOutDectectionCheckClock()
{
    if( TIMEOUTCLOCK_OFF == m_eTimeOutDectectionState )
    {
        return false;
    }

    SYSTEMTIME nowTime;
    GetLocalTime(&nowTime);
    long int nowTime_ms=((nowTime.wHour*+nowTime.wMinute)*+nowTime.wSecond)*+nowTime.wMilliseconds;

    if( nowTime_ms -  m_cTimeOutDectectionStartTime> m_cTimeOutDectectionThrel)
    {
        m_eTimeOutDectectionState = TIMEOUTCLOCK_OFF;
        return true;
    }
    return false;
}
/*******************************************************************************
* Function Name  : singleStopwatchPause
* Description    : pause stopwatch and get time (In Milliseconds).
* Input          : None
* Output         : None
* Return         : succeed/failed
*******************************************************************************/
bool TimeOperation::multipleTimeOutDectectionCheckClock()
{
    if( TIMEOUTCLOCK_OFF == m_eTimeOutDectectionState )
    {
        return false;
    }

    SYSTEMTIME nowTime;
    GetLocalTime(&nowTime);
    long int nowTime_ms=(nowTime.wHour*+nowTime.wMinute)*+nowTime.wMilliseconds;

    if( nowTime_ms > m_cTimeOutDectectionThrel)
    {
        return true;
    }
    return false;
}

TimeOperation.h为头文件

#ifndef __TIMEOPERATION_H
#define __TIMEOPERATION_H
#include <windows.h>
#include <iostream>
typedef enum
{
    STOPWATCH_ON=,
    STOPWATCH_PAUSE=,
    STOPWATCH_STOP=,
}StopWatchState;

typedef enum
{
    TIMEOUTCLOCK_OFF = ,
    TIMEOUTCLOCK_ON = ,
}TimeDectionState;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          

class TimeOperation
{
public:
    TimeOperation();
    ~TimeOperation();
    //stopwatch
    bool singularStopwatchRestart();
    bool singularStopwatchPause(SYSTEMTIME &getTime);
    bool singularStopwatchPause(int &ms);
    //timeout detection
    bool TimeOutDectectionSetClock(SYSTEMTIME &setTime);
    bool TimeOutDectectionSetClock(int ms);
    bool singularTimeOutDectectionCheckClock();
    bool multipleTimeOutDectectionCheckClock();
private:
    //stopwatch
    SYSTEMTIME m_cStopwatchOrigin;
    StopWatchState m_eSingleStopwatchState;
    //TimeOut clock
    long int m_cTimeOutDectectionThrel;
    long int m_cTimeOutDectectionStartTime;
    TimeDectionState m_eTimeOutDectectionState;
};

#endif

main.cpp主程序

#include "main.h"

using namespace cv;
using namespace std;

typedef enum
{
    Step_ReadCammer = ,
    Step_CorrectScreen = ,
    Step_MatchFeature = ,
    Step_Communication = ,
    Step_FastReadCammer =,
    Step_FastCorrectScreen =,
}mainFunctionStep;

vector<Mat> vecFeature;
vector<Mat> vecConfusingFeature;

void preconditioning(void)
{
    string Suffix = ".png";

    for(int i=;i<;i++)
    {
        //int 2 str
        stringstream ss;
        string index;
        ss<<i;
        ss>>index;

        string name = index+Suffix;

        Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE);
        //Mat Img=imread(name);
        if( !Img.data )
        {
            break;
        }
        vecFeature.push_back(Img);//添加灰度特征
    }    

    for(int i=;i<;i++)
    {
        //int 2 str
        stringstream ss;
        string index;
        ss<<i;
        ss>>index;

        string name = index+Suffix;

        Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE);
        //Mat Img=imread(name);
        if( !Img.data )
        {
            break;
        }
        vecConfusingFeature.push_back(Img);//添加灰度特征
    }    

}

int main()
{
    Mat g_srcImage;
    Mat g_extractImage;
    float distance;
    SerialPort *c_COM3;
    mainFunctionStep step = Step_ReadCammer;
    //mainFunctionStep step  = Step_Communication;

    //创建校正对象
    ScreenExtract Screen;

    //加载图片
    preconditioning();

    //打开串口
    try
    {
        c_COM3 = new SerialPort(,);
    }
    catch(char *s)
    {
        return -;
    }

    //打开摄像头
    VideoCapture cap();
    if(!cap.isOpened())
    {
        return -;
    }
    cap.set(CV_CAP_PROP_FRAME_WIDTH, CammerWidth);
    cap.set(CV_CAP_PROP_FRAME_HEIGHT, CammerHeigh);  

    while()
    {
        if( Step_ReadCammer == step )
        {
            //【0】读取测试图片
            //g_srcImage = imread("2.jpg");
            //if(!g_srcImage.data)
            //{
            //    printf("Err");
            //}
            ////namedWindow("【原始图】");
            ////imshow("【原始图】",g_srcImage);

            //【0】读取摄像头
            while()
            {
                cap>>g_srcImage;
                imshow("vedio",g_srcImage);
                if((char(waitKey())=='q'))
                {
                    break;
                }
            }
            step = Step_CorrectScreen;
        }
        else if( Step_CorrectScreen == step)
        {
            //【1】屏幕提取矫正
            Screen.setSrc(g_srcImage);
            Screen.runExtract();
            try
            {
                g_extractImage = Screen.getDst();
            }
            catch(char *s)
            {
                if(s == "Extract fail")//提取失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            imshow("结果",g_extractImage);
            step = Step_MatchFeature;
        }
        else if( Step_MatchFeature == step )
        {
            //【2】特征提取匹配
            Mat featureImage1 = imread("小人无背景.png");
            if(!featureImage1.data)
            {
                printf("featureImage1");
                return ;
            }
            //imshow("素材",featureImage1);

            ImageMatch match(g_extractImage);
            //加入素材特征
            match.setPersonFeature( featureImage1 );
            match.addCubeFeatureImage(vecFeature);
            match.addConfusingFeatureImage(vecConfusingFeature);

            //运行匹配
            try
            {
                match.runMatch();
            }
            catch(char *s)
            {
                if(s == "Person match fail")//小人匹配失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
                else if(s == "Block match fail")//方块匹配失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            distance = match.getDistance();

            step = Step_Communication;
        }
        else if( Step_Communication == step)
        {
            //for(;;)
            //{
            //    int ms;
            //    TimeOperation time;
            //    time.singularStopwatchRestart();
            //    c_COM3->packAndsend(5000);
            //    waitKey(1000);
            //    time.singularStopwatchPause(ms);
            //    cout<<"ms"<<ms<<endl;
            //}
            c_COM3->packAndsend((unsigned short)distance);
            step = Step_FastReadCammer;
            waitKey();
        }
        else if( Step_FastReadCammer == step)
        {
            TimeOperation time;
            time.TimeOutDectectionSetClock(*distance+);
            //【0】读取摄像头
            while()
            {
                cap>>g_srcImage;
                imshow("vedio",g_srcImage);
                if( time.singularTimeOutDectectionCheckClock())
                {
                    break;
                }
                else if((char(waitKey())=='q'))
                {
                    break;
                }
            }
            step = Step_FastCorrectScreen;
        }
        else if( Step_FastCorrectScreen == step)
        {
            //【1】屏幕提取矫正
            Screen.runFastExtract(g_srcImage);
            try
            {
                g_extractImage = Screen.getDst();
            }
            catch(char *s)
            {
                if(s == "Extract fail")//提取失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            imshow("结果",g_extractImage);
            step = Step_MatchFeature;
        }
    }
    while((char(waitKey())!='q'))
    {}
    return ;
}

main.h头文件

#ifndef __MAIN_H
#define __MAIN_H
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include "ScreenExtraction.h"
#include "ImageMatch.h"
#include "SerialPort.h"
#include "TimeOperation.h"

#define DEBUG_SHOW_MAIN
#define CammerWidth 1280
#define CammerHeigh 720

#endif

模板匹配的素材下载：

链接：https://pan.baidu.com/s/1c3YXygC 密码：oeal

---

---

三、关键程序记录

3.1主程序的步骤

1、预处理（加载图片用于模板匹配）

void preconditioning(void)
{
    string Suffix = ".png";

    for(int i=;i<;i++)
    {
        //int 2 str
        stringstream ss;
        string index;
        ss<<i;
        ss>>index;

        string name = index+Suffix;

        Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE);
        //Mat Img=imread(name);
        if( !Img.data )
        {
            break;
        }
        vecFeature.push_back(Img);//添加灰度特征
    }    

    for(int i=;i<;i++)
    {
        //int 2 str
        stringstream ss;
        string index;
        ss<<i;
        ss>>index;

        string name = index+Suffix;

        Mat Img=imread(name,CV_LOAD_IMAGE_GRAYSCALE);
        //Mat Img=imread(name);
        if( !Img.data )
        {
            break;
        }
        vecConfusingFeature.push_back(Img);//添加灰度特征
    }    

}

2、打开摄像头、串口

    //打开串口
    try
    {
        c_COM3 = new SerialPort(,);
    }
    catch(char *s)
    {
        return -;
    }

    //打开摄像头
    VideoCapture cap();
    if(!cap.isOpened())
    {
        return -;
    }
    cap.set(CV_CAP_PROP_FRAME_WIDTH, CammerWidth);
    cap.set(CV_CAP_PROP_FRAME_HEIGHT, CammerHeigh);  

3、主循环部分，在以下5种状态中切换

typedef enum
{
    Step_ReadCammer = ,
    Step_CorrectScreen = ,
    Step_MatchFeature = ,
    Step_Communication = ,
    Step_FastReadCammer =,
    Step_FastCorrectScreen =,
}mainFunctionStep;

• Step_ReadCammer：读取摄像头，在此状态中读取摄像头数据显示到屏幕上，等待用户开始程序：
• Step_CorrectScreen：矫正屏幕，在此状态中进行屏幕的边缘识别、屏幕内容提取矫正等操作：
• Step_MatchFeature：匹配特征，在此状态中进行识别操作，识别人物、方块
• Step_Communication ：进行通信，告诉下位机需要点击多长时间
• Step_FastReadCammer：快速读取摄像头，此步骤与第一步基本相同，不过不需要在进行人为干预，程序会快速读取图像进行处理
• Step_FastCorrectScreen：快速矫正屏幕，此步骤与第二步相似，但不再做边缘识别角点检测等操作，会跟据第二步的记录数据进行快速矫正

while()
    {
        if( Step_ReadCammer == step )
        {
            //【0】读取测试图片
            //g_srcImage = imread("2.jpg");
            //if(!g_srcImage.data)
            //{
            //    printf("Err");
            //}
            ////namedWindow("【原始图】");
            ////imshow("【原始图】",g_srcImage);

            //【0】读取摄像头
            while()
            {
                cap>>g_srcImage;
                imshow("vedio",g_srcImage);
                if((char(waitKey())=='q'))
                {
                    break;
                }
            }
            step = Step_CorrectScreen;
        }
        else if( Step_CorrectScreen == step)
        {
            //【1】屏幕提取矫正
            Screen.setSrc(g_srcImage);
            Screen.runExtract();
            try
            {
                g_extractImage = Screen.getDst();
            }
            catch(char *s)
            {
                if(s == "Extract fail")//提取失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            imshow("结果",g_extractImage);
            step = Step_MatchFeature;
        }
        else if( Step_MatchFeature == step )
        {
            //【2】特征提取匹配
            Mat featureImage1 = imread("小人无背景.png");
            if(!featureImage1.data)
            {
                printf("featureImage1");
                return ;
            }
            //imshow("素材",featureImage1);

            ImageMatch match(g_extractImage);
            //加入素材特征
            match.setPersonFeature( featureImage1 );
            match.addCubeFeatureImage(vecFeature);
            match.addConfusingFeatureImage(vecConfusingFeature);

            //运行匹配
            try
            {
                match.runMatch();
            }
            catch(char *s)
            {
                if(s == "Person match fail")//小人匹配失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
                else if(s == "Block match fail")//方块匹配失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            distance = match.getDistance();

            step = Step_Communication;
        }
        else if( Step_Communication == step)
        {
            //for(;;)
            //{
            //    int ms;
            //    TimeOperation time;
            //    time.singularStopwatchRestart();
            //    c_COM3->packAndsend(5000);
            //    waitKey(1000);
            //    time.singularStopwatchPause(ms);
            //    cout<<"ms"<<ms<<endl;
            //}
            c_COM3->packAndsend((unsigned short)distance);
            step = Step_FastReadCammer;
            waitKey();
        }
        else if( Step_FastReadCammer == step)
        {
            TimeOperation time;
            time.TimeOutDectectionSetClock(*distance+);
            //【0】读取摄像头
            while()
            {
                cap>>g_srcImage;
                imshow("vedio",g_srcImage);
                if( time.singularTimeOutDectectionCheckClock())
                {
                    break;
                }
                else if((char(waitKey())=='q'))
                {
                    break;
                }
            }
            step = Step_FastCorrectScreen;
        }
        else if( Step_FastCorrectScreen == step)
        {
            //【1】屏幕提取矫正
            Screen.runFastExtract(g_srcImage);
            try
            {
                g_extractImage = Screen.getDst();
            }
            catch(char *s)
            {
                if(s == "Extract fail")//提取失败
                {
                    step = Step_ReadCammer;
                    continue;
                }
            }

            imshow("结果",g_extractImage);
            step = Step_MatchFeature;
        }
    }

3.2关键处理：屏幕识别与提取

在ScreenExtraction类中实现了屏幕识别与提取

其中runExtract()方式为第一次识别时的操作。

runFastExtract()为第二次及以后操作，使用上一次runExtract()中储存的数据，只会保留透视变换操作。

Mat ScreenExtract::runExtract()
{
    int cannyThrel=;
    int failCNT=;
    while()
    {
        m_MatEdgeImage = EdgeDection(m_MatEdgeImage,cannyThrel);
        m_MatHoughImage = HoughLine(m_MatEdgeImage);
        m_MatCornerImage = CornerHarris(m_MatHoughImage,m_PointPerspectiveSrcBuff);
        if(failCNT>)
        {
            throw("Extract fail");
            return m_MatSrcImage;
        }
        if(m_PointPerspectiveSrcBuff.size() == )
        {
            break;
        }
        else if(m_PointPerspectiveSrcBuff.size()>)
        {
            if(cannyThrel<)
            {
                cannyThrel+=;
            }
            failCNT++;
        }
        else if(m_PointPerspectiveSrcBuff.size()<)
        {
            if(cannyThrel>)
            {
                cannyThrel-=;
            }
            failCNT++;
        }
        waitKey();
    }

    m_MatDstImageLie = perspectiveChange(m_MatSrcImage,m_PointPerspectiveSrcBuff);
    m_MatDstImageStand = rotation(m_MatDstImageLie,);
    return m_MatDstImageStand;
} 

runExtract()思路如下：

• 使用Canny边缘检测绘制摄像头的所有边缘。
• 使用霍夫变换勾勒出4条边,所构成的矩形为屏幕部分。
• 使用角点检测找到图片中的角点，并使用circle（）函数把所有角点变为实心圆形，此时在附近的角点会连成一个很大的点，图中会剩下4个大点（如图红点）。
• 使用边缘检测对上一步的图操作，并求出质心，效果如图绿点。
• 使用warpPerspective（）等函数进行透视变换，还原屏幕。
• 在上述操作中，很容易出现检测失败，屏幕中多了很多点等等情况，当检测失败时，自动调整第一步中Canny的阈值，重新进行边缘检测

3.3关键处理：识别小人和方块

识别函数在ImageMatch类中。

预先使用Photoshop提取小人和方块的图片，一张一张进行对比，选出匹配度比较高的一张，事实证明，这样做太麻烦，这个方法实在太蠢，识别率也低，以后再改成别的。

据不完全统计，游戏中的方块有如下种类，提取了一些比较特征，可以在以下素材图片第三节中的连接中下载

runMatch()函数操作思路：

• 匹配小人，计算小人的脚底坐标(如图红圈)。
• 判断小人处于屏幕的左边还是右边，我们只需要搜索一边，缩小搜索范围减少计算量（例如在下图中，小人站在屏幕的左半区，方块匹配只需要搜索屏幕的右半区）。
• 为了减少匹配时间，将裁减好的图转为灰度图，循环使用模板匹配上面的模板、挑出匹配率高的模板、计算方块表面中心点坐标（如图方框为识别最高的模板，和绿色圈为方块表面的中心点）。在实际匹配中发现有些方块靠的很近会被遮挡（如下图的闹钟左边被档），容易造成匹配失败，所以匹配失败的时候用那些模板的左/右半边进行再次匹配。
• 对于容易出错的模板，单独分开来最后识别，并提高阈值。
• 若上述操作都没高匹配的图片出现，则选择匹配度最高的一个。
• 计算小人和方块的距离（红圈和绿圈的距离）。

3.3其他程序

距离和按压时间的转换：

　　一次函数y=ax+b，实际使用需要根据下位机误差调整

float ImageMatch::getDistance()
{
    float result = m_fDistance*2.1f+;
    if(result<)
        result=;
    return result;
}

计时类TimeOperation.cpp：

通过获取Windows的时间函数GetLocalTime（）为基础，实现两个功能

1. 秒表(红色)：用来计算某部分程序的运行时间，看看用时多久。
2. 等待超时(蓝色)：在串口发送后，需要等待一段时间才进行下一次获取，使用等待超时功能。

 串口类 SerialPort.cpp：

class SerialPort
{
public:
    SerialPort(int CommNumber,int BaudRate = ,int Parity = ,int ByteSize = , int StopBits = ONESTOPBIT);
    ~SerialPort();
public:
    bool send(vector<unsigned char> vecSend);
    bool packAndsend(unsigned short data1);
private:
    HANDLE m_cHCom;
    int m_iComNumber;
    DCB m_cDcb;
    vector<char> m_vecRecived;
};

用于打开关闭串口、数据打包发送等等。使用的通信协议如下（十六进制表示）