车道线检测之lanelines-detection

时间:2022-02-06 15:16:58

本篇博客整个项目源码:github 
NOTE:本文只介绍了基本的车道线检测方法,预测曲率及车辆位置的车道线检测请戳:无人驾驶之高级车道线检测-AdvanceLanefinding_release

前言

本次博客主要分享Udacity自动驾驶纳米学位的第一个项目,车道线检测算法模型的实现。 
本项目主要实现以下几个功能:

  • 在一张图片上检测出车道线,并将其标记成红色
  • 在一段视频上检测出车道线,并将其标记为红色

基本实现思路

我们采用传统的边缘检测的方法,如canny算法检测出轮廓,然后利用Hough变换检测出直线,最后在图片或者视频上画出直线。模型架构图如下所示: 
车道线检测之lanelines-detection
这模型实现期间我们使用了一些小技巧:

  • 去噪
  • 角度滤波器:滤除极小锐角或极大钝角的线段
  • 选取黄色的色调,并用白色代替
  • 在边缘检测前,放大一些特征

具体代码实现

# importing some useful packages
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import cv2
%matplotlib inline
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
#reading in an image
dir_img = 'test_images/'
image = mpimg.imread('test_images/solidWhiteRight.jpg')#RGB
#print out some stats and plotting
print('This image is:',type(image),'with dimensions:',image.shape)
plt.imshow(image)
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
This image is: <class 'numpy.ndarray'> with dimensions: (540, 960, 3)





<matplotlib.image.AxesImage at 0x7f6064cb4860>
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8

车道线检测之lanelines-detection

车道线检测的一些思路

opencv库为我们提供了一些有用的函数: 
cv2.inRange():颜色选择 
cv2.fillpoly():区域选择 
cv2.line():给定一些端点在图片上画出线 
cv2.addWeighted():添加或覆盖两个图像 
cv2.cvtColor():转换成灰度图或者改变图像颜色 
cv2.imwrite():将图片输出到文件 
cv2.bitwise_and():在图片上应用蒙板

接下来我们实现一些辅助函数

import math

def grayscale(img):
    ''' 灰度转换,返回只有一个颜色通道的图像 '''
    return cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)#if you read an image with cv.imread(),return cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

def canny(img,low_threshold,high_threshold):
    return cv2.Canny(img,low_threshold,high_threshold)

def gaussian_blur(img,kernel_size):
    '''Applies a Gaussian Noise Kernel'''#高斯模糊
    return cv2.GaussianBlur(img,(kernel_size,kernel_size),0)

def region_of_interest(img,vertices):
    ''' Apply an image mask '''
    #define a blank mask to start with
    mask = np.zeros_like(img)

    #defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(img.shape)>2:
        channel_count = img.shape[2]
        ignore_mask_color = (255,)channel_count else: ignore_mask_color = 255#涂黑 cv2.fillPoly(mask,vertices,ignore_mask_color) masked_image = cv2.bitwise_and(img,mask)#与操作 return masked_image def draw_lines(img,lines,color=[255,0,0],thickness=2): ''' Note: ''' #print(lines) for line in lines: for x1,y1,x2,y2 in line: cv2.line(img,(x1,y1),(x2,y2),color,thickness) def hpass_angle_filter(lines,angle_threshold): if lines.shape!=None: filtered_lines =[] for line in lines: for x1,y1,x2,y2 in line: angle = abs(np.arctan((y2-y1)/(x2-x1))180/np.pi)
                if angle > angle_threshold:
                    filtered_lines.append([[x1,y1,x2,y2]])
        return filtered_lines
def average_lines(img,lines,y_min,y_max):
    #return coordinates of the averaged lines
    hough_pts = {'m_left':[],'b_left':[],'norm_left':[],'m_right':[],'b_right':[],'norm_right':[]}
    if lines != None:
        for line in lines:
            for x1,y1,x2,y2 in line:
                m,b = np.polyfit([x1,x2],(y1,y2),1)
                norm = ((x2-x1)2+(y2-y1)2)*0.5 if m>0:#斜率right hough_pts['m_right'].append(m) hough_pts['b_right'].append(b) hough_pts['norm_right'].append(norm) if m<0: hough_pts['m_left'].append(m) hough_pts['b_left'].append(b) hough_pts['norm_left'].append(norm) if len(hough_pts['b_left'])!=0 or len(hough_pts['m_left'])!=0 or len(hough_pts['norm_left'])!=0: b_avg_left = np.mean(np.array(hough_pts['b_left'])) m_avg_left = np.mean(np.array(hough_pts['m_left'])) xmin_left = int((y_min-b_avg_left)/m_avg_left) xmax_left = int((y_max-b_avg_left)/m_avg_left) left_lane = [[xmin_left,y_min,xmax_left,y_max]] else: left_lane = [[0,0,0,0]] if len(hough_pts['b_right'])!=0 or len(hough_pts['m_right'])!=0 or len(hough_pts['norm_right'])!=0: b_avg_right = np.mean(np.array(hough_pts['b_right'])) m_avg_right = np.mean(np.array(hough_pts['m_right'])) xmin_right = int((y_min - b_avg_right)/m_avg_right) xmax_right = int((y_max-b_avg_right)/m_avg_right) right_lane = [[xmin_right,y_min,xmax_right,y_max]] else: right_lane = [[0,0,0,0]] return [left_lane,right_lane] def hough_lines(img,rho,theta,threshold,min_line_len,max_line_gap): ''' img is the output of canny tranform return an image with hough lines draw ''' lines = cv2.HoughLinesP(img,rho,theta,threshold,np.array([]),minLineLength=min_line_len,maxLineGap = max_line_gap) #print(lines.shape) #line_img = np.zeros((img.shape[0],img.shape[1],3),dtype=np.uint8) ''' line2 = [] try: for line in lines: for x1,y1,x2,y2 in line: if abs(y1-y2)<10: continue k = float(y2-y1)/(x2-x1) if y1>y2: extend = int(x2+(img.shape[0]-y2)/k) lines2.append([x2-x1,y1,k,extend]) lines2 = np.array(lines2) lines3 = [] for side in [lines2[lines2[:,2]<0],lines2[lines2[:,2]>0]]: h2 = side[:,1].min() side[:,0]/=side[:,0].min() k1 = np.average(side[:,2],weights=side[:,0]) x1 = np.average(side[:,3],weights=side[:,0]) lines3.append([int(x1),img.shape[0],int(x1-(img.shape[0]-h2)/k1),int(h2)]) arr+=np.array(lines3) except: pass lines4 = arr.data.mean(axis=0) draw_lines(line_img,[lines4]) ''' #draw_lines(line_img,lines) return lines def weighted_img(img,initial_img,a=0.8,b=1,r=0.0): return cv2.addWeighted(initial_img,a,img,b,r)
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
  • 70
  • 71
  • 72
  • 73
  • 74
  • 75
  • 76
  • 77
  • 78
  • 79
  • 80
  • 81
  • 82
  • 83
  • 84
  • 85
  • 86
  • 87
  • 88
  • 89
  • 90
  • 91
  • 92
  • 93
  • 94
  • 95
  • 96
  • 97
  • 98
  • 99
  • 100
  • 101
  • 102
  • 103
  • 104
  • 105
  • 106
  • 107
  • 108
  • 109
  • 110
  • 111
  • 112
  • 113
  • 114
  • 115
  • 116
  • 117
  • 118
  • 119
  • 120
  • 121
  • 122
  • 123
  • 124
  • 125
  • 126

test images

接下来,我们来对一些图片做个测试

import os
dir_img = 'test_images/'
f = os.listdir(dir_img)
  • 1
  • 2
  • 3
f = [fname for fname in f if 'jpeg' in fname or 'jpg' in fname] def show_img(ax,img,cmap,title): if cmap=='gray': ax.imshow(img,cmap='gray') else: ax.imshow(img) ax.axis('off') ax.set_title(title) def pipeline(img,vertices,threshold_angle,hline_show): ''' pipeline: ''' # convert to grayscale gray = grayscale(img) # Apply Gaussian Blur:noise smooth noise gray_blur = gaussian_blur(gray,3) # Apply canny edge detector edges = canny(gray_blur,10,180)#这个阈值的选取,主要靠经验 # apply mask  imshape = img.shape masked = region_of_interest(edges,vertices) h_lines = hough_lines(masked,rho=1,theta=np.pi/180,threshold=25,min_line_len=10,max_line_gap=10)#检测直线 #Hough Tramsform lines if hline_show['hlines']=='on': hlines_img = np.zeros(imshape,dtype=np.uint8) draw_lines(hlines_img,h_lines,color=[255,0,0],thickness=2) else: hlines_img = np.zeros(imshape,dtype=np.uint8) #Angle High Pass filter h_lines = hpass_angle_filter(h_lines,threshold_angle) # average lines if hline_show['avg']=='on': avg_hlines = average_lines(img,h_lines,int(img.shape[0]0.65),img.shape[0]) avg_img = np.zeros(imshape,dtype=np.uint8) draw_lines(avg_img,avg_hlines,color=[255,0,0],thickness=10) else: avg_img = np.zeros(imshape,dtype=np.uint8) #Display result of each step of the pipeline if hline_show['steps']=='on': ,ax = plt.subplots(2,3,figsize=(20,10))
        showimg(ax[0,0],img,None,'original_img') show_img(ax[0,1],gray,'gray','Apply grayscale') show_img(ax[0,2],gray_blur,'gray','Apply Gaussian Blur') show_img(ax[1,0],edges,'gray','Apply Canny') show_img(ax[1,1],masked,'gray','Apply mask') show_img(ax[1,2],hlines_img,None,'Apply Hough') plt.show() img_all_lines = weighted_img(hlines_img,img,a=1,b=0.8,r=0.0)#计算两个矩阵的权重和 img_all_lines = weighted_img(avg_img,img_all_lines,a=1,b=0.8,r=0.0) return img_all_lines for img_name in f: #reading in an imag print('Image:',img_name) img = mpimg.imread(dir_img+img_name) hline_show = {'hlines':'on','avg':'off','steps':'on'} imshape = img.shape # vertices 圈定了车道线的范围 vertices = np.array([[(100,imshape[0]),(390,imshape[0]0.65),\ (620,imshape[0]0.65),(imshape[1],imshape[0]),\ (100,imshape[0])]],dtype=np.int32) threshold_angle = 25 lines_img = pipeline(img,vertices,threshold_angle,hline_show) plt.imshow(lines_img) plt.show()
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
Image: solidYellowCurve2.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

Image: solidYellowCurve.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

Image: solidWhiteCurve.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

Image: solidWhiteRight.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

Image: whiteCarLaneSwitch.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

Image: solidYellowLeft.jpg
  • 1
  • 2

车道线检测之lanelines-detection

车道线检测之lanelines-detection

test on video

接下来我们来看看在视频上处理的效果 
我们需要在视频上画出车道线

import imageio
#imageio.plugins.ffmpeg.download()
from moviepy.editor import VideoFileClip
from IPython.display import HTML
  • 1
  • 2
  • 3
  • 4
def process_image(image):
    #返回值必须是一个三通道的彩色图像
    hline_show = {'hlines':'on','avg':'on','steps':'off'}
    imshape = img.shape
    vertices = np.array([[(100,imshape[0]),(390,imshape[0]0.65),\ (620,imshape[0]0.65),(imshape[1],imshape[0]),\
                         (100,imshape[0])]],dtype=np.int32)
    threshold_angle = 25
    return pipeline(image,vertices,threshold_angle,hline_show)
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
dir_video = 'test_videos/'
white_output = dir_video+'white.mp4'
clip1 = VideoFileClip(dir_video+'solidWhiteRight.mp4')
white_clip = clip1.fl_image(process_image)
%time white_clip.write_videofile(white_output,audio=False)
  • 1
  • 2
  • 3
  • 4
  • 5
[MoviePy] >>>> Building video test_videos/white.mp4
[MoviePy] Writing video test_videos/white.mp4


100%|█████████▉| 221/222 [00:02<00:00, 77.20it/s]


[MoviePy] Done.
[MoviePy] >>>> Video ready: test_videos/white.mp4 

CPU times: user 1.99 s, sys: 125 ms, total: 2.12 s
Wall time: 3.07 s
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
HTML(''' <video width='960' height='540' controls> <source src =white_output> </video> ''')
  • 1
  • 2
  • 3
  • 4
  • 5



Imporvement

接下来我们来看看一些值得改善与提高的地方: 
我们知道,车道线检测算法的性能受道路上的一些“噪声”的影响,比如说阴影、道路损坏等。 
有些断断续续的路段可以通过高通滤波器来连接。但是算法并不适用于跟踪弯曲车道线 
因此我们从以下三个方面来提高算法的性能:

  • 角度滤波器:滤除极小锐角或极大钝角的线段
  • 选取黄色的色调,并用白色代替
  • 在边缘检测前,放大一些特征
def bypass_angle_filter(lines,low_thres,hi_thres): ''' 前面的代码中,我们也实现了一个角度滤波器,但是这两个函数还是有区别的, 前面的函数只有一个阈值,而这个函数有两个阈值,并只保留阈值之间的角度 ''' filtered_lines = [] for line in lines: for x1,y1,x2,y2 in line: angle = abs(np.arctan((y2-y1)/(x2-x1))180/np.pi) if angle > low_thres and angle < hi_thres: filtered_lines.append([[x1,y1,x2,y2]]) return filtered_lines def yellow_enhance(img_rgb): ''' 该函数将rgb中淡黄色转换成白色,主要分三步: step1:convert rgb to hsv step2:create a lower/upper range of hsv step3:create a mask ''' img_hsv = cv2.cvtColor(img_rgb,cv2.COLOR_BGR2HSV) lower_yellow = np.array([40,100,20]) upper_yellow = np.array([100,255,255]) mask = cv2.inRange(img_hsv,lower_yellow,upper_yellow) gray = grayscale(img_rgb) return weighted_img(mask,gray,a=1.,b=1.,r=0.) def pipeline(img,vertices,low_thres,hi_thres,hline_show): #convert to grayscale + enhance yellow-ish tone gray = yellow_enhance(img) #remove /cleanup noise gray_blur = gaussian_blur(gray,3) # dilate features for large gap between edges line gray =cv2.dilate(gray,(3,3),iterations=10) edges = canny(gray,50,180) imshape = img.shape masked = region_of_interest(edges,vertices) h_lines = hough_lines(masked,rho=1,theta=np.pi/180,threshold=26,min_line_len=5,max_line_gap=50) if hline_show['hlines']=='on': hlines_img = np.zeros(imshape,dtype=np.uint8) draw_lines(hlines_img,h_lines,color=[255,0,0],thickness=4) else: hlines_img = np.zeros(imshape,dtype=np.uint8) # Angle High Pass filter h_lines = bypass_angle_filter(h_lines,low_thres,hi_thres) #averaging lines if hline_show['avg'] == 'on': avg_hlines = average_lines(img,h_lines,int(img.shape[0]0.65),img.shape[0]) avg_img = np.zeros(imshape,dtype=np.uint8) draw_lines(avg_img,avg_hlines,color=[255,0,0],thickness=10) else: avg_img = np.zeros(imshape,dtype=np.uint8) #Display result of each step of the pipeline if hline_show['steps']=='on': ,ax = plt.subplots(2,3,figsize=(20,10))
        show_img(ax[0,0],img,None,'original_img')
        show_img(ax[0,1],gray,'gray','Apply grayscale')
        show_img(ax[0,2],gray_blur,'gray','Apply Gaussian Blur')
        show_img(ax[1,0],edges,'gray','Apply Canny')
        show_img(ax[1,1],masked,'gray','Apply mask')
        show_img(ax[1,2],hlines_img,None,'Apply Hough')
        plt.show()
    img_all_lines = weighted_img(hlines_img,img,a=1.,b=0.8,r=0.)
    img_all_lines = weighted_img(avg_img,img_all_lines,a=1.,b=0.8,r=0.)
    return img_all_lines
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13
  • 14
  • 15
  • 16
  • 17
  • 18
  • 19
  • 20
  • 21
  • 22
  • 23
  • 24
  • 25
  • 26
  • 27
  • 28
  • 29
  • 30
  • 31
  • 32
  • 33
  • 34
  • 35
  • 36
  • 37
  • 38
  • 39
  • 40
  • 41
  • 42
  • 43
  • 44
  • 45
  • 46
  • 47
  • 48
  • 49
  • 50
  • 51
  • 52
  • 53
  • 54
  • 55
  • 56
  • 57
  • 58
  • 59
  • 60
  • 61
  • 62
  • 63
  • 64
  • 65
  • 66
  • 67
  • 68
  • 69
def process_image(image):
    hline_show = {'hlines':'off','avg':'on','steps':'off'}
    imshape = [720,1280]
    vertices = np.array([[(200,imshape[0]-80),(490,imshape[0]0.65),\ (820,imshape[0]0.65),(imshape[1]-150,imshape[0]-80),\
                         (100,imshape[0]-80)]],dtype=np.int32)
    low_thres,hi_thres = [30,80]
    return pipeline(image,vertices,low_thres,hi_thres,hline_show)
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
challenge_output = dir_video +'extra.mp4'
clip2 = VideoFileClip(dir_video+'challenge.mp4')
challenge_clip = clip2.fl_image(process_image)
%time challenge_clip.write_videofile(challenge_output,audio=False)
  • 1
  • 2
  • 3
  • 4
[MoviePy] >>>> Building video test_videos/extra.mp4
[MoviePy] Writing video test_videos/extra.mp4


100%|██████████| 251/251 [00:06<00:00, 37.31it/s]


[MoviePy] Done.
[MoviePy] >>>> Video ready: test_videos/extra.mp4 

CPU times: user 5.55 s, sys: 213 ms, total: 5.77 s
Wall time: 7.43 s
  • 1
  • 2
  • 3
  • 4
  • 5
  • 6
  • 7
  • 8
  • 9
  • 10
  • 11
  • 12
  • 13

无人驾驶之高级车道线检测-AdvanceLane_finding_release