本文以2.6.32.68内核中的mpt2sas为例子，介绍了sas驱动的设备管理。
1. 基本结构
内核中scsi的结构分三层，此在网上已有大量资料，不再赘述。本文在此基础上增加了mid layer的 transport描述。如图1-1，所示，transport_sas也属于中层一部分，其主要作用在于通过sysfs向用户提供整个拓扑描述及对拓扑中设备的管理（提供发送SMP的消息接口）
图1-1 基本结构图
2. 基本数据结构
2.1 mid layer 数据结构
\linux-2.6.32.68\include\scsi\scsi_device.h
struct scsi_device
表示一个scsi设备，此数据结构用于和upper layer层的驱动打交道。mid layer从lower layer 获取某个物理设备的信息后，填充此数据接口，而后调用scsi_sysfs_add_sdev(sdev)，将此设备挂接到scsi总线上，通知upper layer层驱动struct scsi_target
表示一个scsi target，用于mid layer 和lower layer打交道。2.2 transport_sas数据结构
\linux-2.6.32.68\include\scsi\ scsi_transport_sas.h
2.2.1 phy相关
如图2-1所示的青色部分，用struct sas_phy结构表示；紫色部分，用struct sas_rphy表示。
图2-1 rphy和phy概念图
* the SAS remote PHYrepresented by
* struct sas_rphy defines an"incoming" PHY on a SAS Expander or
* end device.
struct sas_rphy {
struct device dev;
struct sas_identify identify;
struct list_head list;
struct request_queue *q;
u32 scsi_target_id;
};
struct sas_phy {
struct device dev;
int number;
int enabled;
/* phy identification */
struct sas_identify identify;
/* phy attributes */
enum sas_linkrate negotiated_linkrate;
enum sas_linkrate minimum_linkrate_hw;
enum sas_linkrate minimum_linkrate;
enum sas_linkrate maximum_linkrate_hw;
enum sas_linkrate maximum_linkrate;
/* link error statistics */
u32 invalid_dword_count;
u32 running_disparity_error_count;
u32 loss_of_dword_sync_count;
u32 phy_reset_problem_count;
/* for the list of phys belonging to a port */
struct list_head port_siblings;
struct work_struct reset_work;
};
以上两个phy相关的结构体，rphy是代表对设备(end device 、expander)的管理；而phy则是对物理phy的管理。
port则是phy的集合；同时和rphy所代表的设备建立联系。
struct sas_port {
struct device dev;
int port_identifier;
int num_phys;
/* port flags */
unsigned int is_backlink:1;
/* the other end of the link */
struct sas_rphy *rphy;
struct mutex phy_list_mutex;
struct list_head phy_list;
};
2.2.2 设备
expander和end device分别用不同结构体表示，而共有rphy这个结构体。
struct sas_expander_device {
int level;
int next_port_id;
#define SAS_EXPANDER_VENDOR_ID_LEN 8
char vendor_id[SAS_EXPANDER_VENDOR_ID_LEN+1];
#define SAS_EXPANDER_PRODUCT_ID_LEN 16
char product_id[SAS_EXPANDER_PRODUCT_ID_LEN+1];
#define SAS_EXPANDER_PRODUCT_REV_LEN 4
char product_rev[SAS_EXPANDER_PRODUCT_REV_LEN+1];
#define SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN 8
char component_vendor_id[SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN+1];
u16 component_id;
u8 component_revision_id;
struct sas_rphy rphy;
};
struct sas_end_device {
struct sas_rphy rphy;
/* flags */
unsigned ready_led_meaning:1;
/* parameters */
u16 I_T_nexus_loss_timeout;
u16 initiator_response_timeout;
};
2.3 lower layer数据结构
和设备相关的，有5个，分别表示phy 、port、end device、expander&host 、及芯片本身
以mpt2sas驱动为例，
linux-2.6.32.68\drivers\scsi\mpt2sas\mpt2sas_base.h
下面结构体中使用了 device handle，此字段为固件为每个设备（例如硬盘、expander）分配的一个数字，每个依次递增，对磁盘进行拔出再插入后，其handle会被重新分配。
<strong> _sas_device</strong>
/**
* struct _sas_device - attached device information
* @list: sas device list
* @starget: starget object
* @sas_address: device sas address
* @device_name: retrieved from the SAS IDENTIFY frame.
* @handle: device handle
* @parent_handle: handle to parent device
* @enclosure_handle: enclosure handle
* @enclosure_logical_id: enclosure logical identifier
* @volume_handle: volume handle (valid when hidden raid member)
* @volume_wwid: volume unique identifier
* @device_info: bitfield provides detailed info about the device
* @id: target id
* @channel: target channel
* @slot: number number
* @hidden_raid_component: set to 1 when this is a raid member
* @responding: used in _scsih_sas_device_mark_responding
*/
struct _sas_device {
struct list_head list;
struct scsi_target *starget;
u64 sas_address;
u64 device_name;
u16 handle;
u16 parent_handle;
u16 enclosure_handle;
u64 enclosure_logical_id;
u16 volume_handle;
u64 volume_wwid;
u32 device_info;
int id;
int channel;
u16 slot;
u8 hidden_raid_component;
u8 responding;
u16 state;
};
<strong> _sas_port</strong>
对端口（由一个或者多个phy组成）进行管理。
/**
* struct _sas_port - wide/narrow sas port information
* @port_list: list of ports belonging to expander
* @handle: device handle for this port
* @sas_address: sas address of this port
* @num_phys: number of phys belonging to this port
* @remote_identify: attached device identification
* @rphy: sas transport rphy object
* @port: sas transport wide/narrow port object
* @phy_list: _sas_phy list objects belonging to this port
*/
struct _sas_port {
struct list_head port_list;
u16 handle;
u64 sas_address;
u8 num_phys;
struct sas_identify remote_identify;
struct sas_rphy *rphy;
struct sas_port *port;
struct list_head phy_list;
};
<strong> _sas_phy</strong>
/**
* struct _sas_phy - phy information
* @port_siblings: list of phys belonging to a port
* @identify: phy identification
* @remote_identify: attached device identification
* @phy: sas transport phy object
* @phy_id: unique phy id
* @handle: device handle for this phy
* @attached_handle: device handle for attached device
*/
struct _sas_phy {
struct list_head port_siblings;
struct sas_identify identify;
struct sas_identify remote_identify;
struct sas_phy *phy;
u8 phy_id;
u16 handle;
u16 attached_handle;
};
<strong> _sas_node</strong>
/**
* struct _sas_node - sas_host/expander information
* @list: list of expanders
* @parent_dev: parent device class
* @num_phys: number phys belonging to this sas_host/expander
* @sas_address: sas address of this sas_host/expander
* @handle: handle for this sas_host/expander
* @parent_handle: parent handle
* @enclosure_handle: handle for this a member of an enclosure
* @device_info: bitwise defining capabilities of this sas_host/expander
* @responding: used in _scsih_expander_device_mark_responding
* @phy: a list of phys that make up this sas_host/expander
* @sas_port_list: list of ports attached to this sas_host/expander
*/
struct _sas_node {
struct list_head list;
struct device *parent_dev;
u8 num_phys;
u64 sas_address;
u16 handle;
u16 parent_handle;
u16 enclosure_handle;
u64 enclosure_logical_id;
u8 responding;
struct _sas_phy *phy;
struct list_head sas_port_list;
};
3 设备的添加
以热插入一个设备为例子，描述了一个设备添加过程中，第二章中相关结构体的创建及彼此建立连接。
3.1 热插入事件处理
设备插入后，ioc会通过中断通知驱动，驱动根据中断携带的事件（expander添加、target添加）等进行不同的处理
/**
* _scsih_sas_topology_change_event - handle topology changes
* @ioc: per adapter object
* @fw_event: The fw_event_work object
* Context: user.
*
*/
static void
_scsih_sas_topology_change_event(struct MPT2SAS_ADAPTER *ioc,
struct fw_event_work *fw_event)
{
int i;
u16 parent_handle, handle;
u16 reason_code;
u8 phy_number;
struct _sas_node *sas_expander;
unsigned long flags;
u8 link_rate_;
Mpi2EventDataSasTopologyChangeList_t *event_data = fw_event->event_data;
#ifdef CONFIG_SCSI_MPT2SAS_LOGGING
if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
_scsih_sas_topology_change_event_debug(ioc, event_data);
#endif
if (!ioc->sas_hba.num_phys)
_scsih_sas_host_add(ioc);
else
_scsih_sas_host_refresh(ioc, 0);
if (fw_event->ignore) {
dewtprintk(ioc, printk(MPT2SAS_DEBUG_FMT "ignoring expander "
"event\n", ioc->name));
return;
}
parent_handle = le16_to_cpu(event_data->ExpanderDevHandle);
/* handle expander add */
if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_ADDED)
if (_scsih_expander_add(ioc, parent_handle) != 0) /* 博主：1. 针对IOC固件上报上来的expander添加事件，调用expander添加接口 */
return;
/* handle siblings events */
for (i = 0; i < event_data->NumEntries; i++) {
if (fw_event->ignore) {
dewtprintk(ioc, printk(MPT2SAS_DEBUG_FMT "ignoring "
"expander event\n", ioc->name));
return;
}
if (ioc->shost_recovery)
return;
phy_number = event_data->StartPhyNum + i;
reason_code = event_data->PHY[i].PhyStatus &
MPI2_EVENT_SAS_TOPO_RC_MASK;
if ((event_data->PHY[i].PhyStatus &
MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT) && (reason_code !=
MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING))
continue;
handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
if (!handle)
continue;
link_rate_ = event_data->PHY[i].LinkRate >> 4;
switch (reason_code) {
case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
if (!parent_handle) {
if (phy_number < ioc->sas_hba.num_phys)
mpt2sas_transport_update_links(
ioc,
ioc->sas_hba.phy[phy_number].handle,
handle, phy_number, link_rate_);
} else {
spin_lock_irqsave(&ioc->sas_node_lock, flags);
sas_expander =
mpt2sas_scsih_expander_find_by_handle(ioc,
parent_handle);
spin_unlock_irqrestore(&ioc->sas_node_lock,
flags);
if (sas_expander) {
if (phy_number < sas_expander->num_phys)
mpt2sas_transport_update_links(
ioc,
sas_expander->
phy[phy_number].handle,
handle, phy_number,
link_rate_);
}
}
if (reason_code == MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED) { /* 博主：2. end device 添加事件，添加设备*/
if (link_rate_ < MPI2_SAS_NEG_LINK_RATE_1_5)
break;
_scsih_add_device(ioc, handle, phy_number, 0);
}
break;
case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
_scsih_remove_device(ioc, handle);
break;
}
}
/* handle expander removal */
if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
_scsih_expander_remove(ioc, parent_handle);
}
3.2 expander的添加
expander添加的代码如下：先创建和其相连的上行设备的端口并添加到transport中，最后添加expander的phy。
/**
* _scsih_expander_add - creating expander object
* @ioc: per adapter object
* @handle: expander handle
*
* Creating expander object, stored in ioc->sas_expander_list.
*
* Return 0 for success, else error.
*/
static int
_scsih_expander_add(struct MPT2SAS_ADAPTER *ioc, u16 handle)
{
struct _sas_node *sas_expander;
Mpi2ConfigReply_t mpi_reply;
Mpi2ExpanderPage0_t expander_pg0;
Mpi2ExpanderPage1_t expander_pg1;
Mpi2SasEnclosurePage0_t enclosure_pg0;
u32 ioc_status;
u16 parent_handle;
__le64 sas_address;
int i;
unsigned long flags;
struct _sas_port *mpt2sas_port = NULL;
int rc = 0;
if (!handle)
return -1;
if (ioc->shost_recovery)
return -1;
if ((mpt2sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
MPI2_SAS_EXPAND_PGAD_FORM_HNDL, handle))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
MPI2_IOCSTATUS_MASK;
if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
/* handle out of order topology events */
parent_handle = le16_to_cpu(expander_pg0.ParentDevHandle);
if (parent_handle >= ioc->sas_hba.num_phys) {
spin_lock_irqsave(&ioc->sas_node_lock, flags);
sas_expander = mpt2sas_scsih_expander_find_by_handle(ioc,
parent_handle);
spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
if (!sas_expander) {
rc = _scsih_expander_add(ioc, parent_handle);
if (rc != 0)
return rc;
}
}
spin_lock_irqsave(&ioc->sas_node_lock, flags);
sas_address = le64_to_cpu(expander_pg0.SASAddress);
sas_expander = mpt2sas_scsih_expander_find_by_sas_address(ioc,
sas_address);
spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
if (sas_expander)
return 0;
/* 博主：此处分配了一个_sas_node的结构体，按之前所描述，此结构体可以用于表示一个expander设备 */
sas_expander = kzalloc(sizeof(struct _sas_node),
GFP_KERNEL);
if (!sas_expander) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
<pre name="code" class="cpp">
sas_expander->handle = handle;
sas_expander->num_phys = expander_pg0.NumPhys; /* 博主:expander 的phy数目 */
sas_expander->parent_handle = parent_handle;
sas_expander->enclosure_handle =
le16_to_cpu(expander_pg0.EnclosureHandle);
sas_expander->sas_address = sas_address;
printk(MPT2SAS_INFO_FMT "expander_add: handle(0x%04x),"
" parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n", ioc->name,
handle, sas_expander->parent_handle, (unsigned long long)
sas_expander->sas_address, sas_expander->num_phys);
if (!sas_expander->num_phys)
goto out_fail;
sas_expander->phy = kcalloc(sas_expander->num_phys,
sizeof(struct _sas_phy), GFP_KERNEL); /* 博主:此处一次性申请此expander 的所有phy的结构体,N个_sas_phy,并用expander结构体中的phy指针指向此段连续的地址。 */
if (!sas_expander->phy) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
rc = -1;
goto out_fail;
}
INIT_LIST_HEAD(&sas_expander->sas_port_list); /* 初始化expander的port链表，准备为expander的port建立关联 */
mpt2sas_port = mpt2sas_transport_port_add(ioc, handle,
sas_expander->parent_handle); /* 1.1 创建了expander设备后，或者expander物理设备被插入后，和expander相连接的上行设备（expander或者HBA）
就明确了本身有多少phy和新进入系统的expander相连接，因而就可以创建port，此处即为创建新插入设备的port，并添加到系统中。因而port实际上是一个上行设备的
下行端口。 */
if (!mpt2sas_port) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
rc = -1;
goto out_fail;
}
sas_expander->parent_dev = &mpt2sas_port->rphy->dev;
/* 1.2 expander本身添加到系统后，下面处理expander的每个phy */
for (i = 0 ; i < sas_expander->num_phys ; i++) {
if ((mpt2sas_config_get_expander_pg1(ioc, &mpi_reply,
&expander_pg1, i, handle))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
rc = -1;
goto out_fail;
}
sas_expander->phy[i].handle = handle;
sas_expander->phy[i].phy_id = i;
if ((mpt2sas_transport_add_expander_phy(ioc,
&sas_expander->phy[i], expander_pg1,
sas_expander->parent_dev))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
rc = -1;
goto out_fail;
}
}
if (sas_expander->enclosure_handle) {
if (!(mpt2sas_config_get_enclosure_pg0(ioc, &mpi_reply,
&enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
sas_expander->enclosure_handle))) {
sas_expander->enclosure_logical_id =
le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
}
}
_scsih_expander_node_add(ioc, sas_expander); /* 最终将创建的expander设备，连入到ioc上 */
return 0;
out_fail:
if (mpt2sas_port)
mpt2sas_transport_port_remove(ioc, sas_expander->sas_address,
sas_expander->parent_handle);
kfree(sas_expander);
return rc;
}
3.3 端口的添加
此接口expander、end device新增时，都会被调用。
创建port的流程如下：expander新添加后，创建和其相连的上行端口的port，并将此端口sas_port 及端口里面的phy sas_phy，及下挂的设备rphy注册到transport层。对于expander设备，通过ioc下发SMP命令获取expander的厂商信息。
</pre><pre>
/**
* mpt2sas_transport_port_add - insert port to the list
* @ioc: per adapter object
* @handle: handle of attached device
* @parent_handle: parent handle(either hba or expander)
* Context: This function will acquire ioc->sas_node_lock.
*
* Adding new port object to the sas_node->sas_port_list.
*
* Returns mpt2sas_port.
*/
struct _sas_port *
mpt2sas_transport_port_add(struct MPT2SAS_ADAPTER *ioc, u16 handle,
u16 parent_handle)
{
struct _sas_phy *mpt2sas_phy, *next;
struct _sas_port *mpt2sas_port;
unsigned long flags;
struct _sas_node *sas_node;
struct sas_rphy *rphy;
int i;
struct sas_port *port;
if (!parent_handle)
return NULL;
/* 博主：申请_sas_port的内存*/
mpt2sas_port = kzalloc(sizeof(struct _sas_port),
GFP_KERNEL);
if (!mpt2sas_port) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return NULL;
}
INIT_LIST_HEAD(&mpt2sas_port->port_list);
INIT_LIST_HEAD(&mpt2sas_port->phy_list);
spin_lock_irqsave(&ioc->sas_node_lock, flags);
sas_node = _transport_sas_node_find_by_handle(ioc, parent_handle);
spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
if (!sas_node) {
printk(MPT2SAS_ERR_FMT "%s: Could not find parent(0x%04x)!\n",
ioc->name, __func__, parent_handle);
goto out_fail;
}
mpt2sas_port->handle = parent_handle;
mpt2sas_port->sas_address = sas_node->sas_address;
if ((_transport_set_identify(ioc, handle,
&mpt2sas_port->remote_identify))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
goto out_fail;
}
if (mpt2sas_port->remote_identify.device_type == SAS_PHY_UNUSED) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
goto out_fail;
}
/* 将expander或者HBA（即_sas_node）中，sas address相同的phy添加到同一个port中 */
for (i = 0; i < sas_node->num_phys; i++) {
if (sas_node->phy[i].remote_identify.sas_address !=
mpt2sas_port->remote_identify.sas_address)
continue;
list_add_tail(&sas_node->phy[i].port_siblings,
&mpt2sas_port->phy_list);
mpt2sas_port->num_phys++;
}
if (!mpt2sas_port->num_phys) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
goto out_fail;
}
/*博主： 此处分别sas_port内存，并将此sas_port注册到sas_transport class中 */
port = sas_port_alloc_num(sas_node->parent_dev);
if ((sas_port_add(port))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
goto out_fail;
}
/* 博主：此处将同一个_sas_port中的phy，依次添加到sas_transport class中的sas_port中，即将phy注册到sas_transport class中 */
list_for_each_entry(mpt2sas_phy, &mpt2sas_port->phy_list,
port_siblings) {
if ((ioc->logging_level & MPT_DEBUG_TRANSPORT))
dev_printk(KERN_INFO, &port->dev, "add: handle(0x%04x)"
", sas_addr(0x%016llx), phy(%d)\n", handle,
(unsigned long long)
mpt2sas_port->remote_identify.sas_address,
mpt2sas_phy->phy_id);
sas_port_add_phy(port, mpt2sas_phy->phy);
}
/*博主：此处建立transport 层的sas_port和Lower layer层的_sas_port的关联 */
mpt2sas_port->port = port;
/* 根据设备类型，分别创建sas_tranport class所需的 sas_device 或者sas_expander,两者都作为一个rphy！ */
if (mpt2sas_port->remote_identify.device_type == SAS_END_DEVICE)
rphy = sas_end_device_alloc(port);
else
rphy = sas_expander_alloc(port,
mpt2sas_port->remote_identify.device_type);
/* 博主： 新建port下挂的设备，即为rphy所代表的设备。如前所属，rphy表示一个下行的设备，当有expander或者
磁盘新插入时，需要生成此结构来表示 */
rphy->identify = mpt2sas_port->remote_identify;
/*博主： 此处将新发现的设备添加到transport层，并最终调用mid layer中的接口，向系统注册设备 */
if ((sas_rphy_add(rphy))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
}
if ((ioc->logging_level & MPT_DEBUG_TRANSPORT))
dev_printk(KERN_INFO, &rphy->dev, "add: handle(0x%04x), "
"sas_addr(0x%016llx)\n", handle,
(unsigned long long)
mpt2sas_port->remote_identify.sas_address);
mpt2sas_port->rphy = rphy; /* 博主：下挂设备创建完毕 */
spin_lock_irqsave(&ioc->sas_node_lock, flags);
list_add_tail(&mpt2sas_port->port_list, &sas_node->sas_port_list);/*博主: 最终将设备添加到_sas_node所代表设备的链表中 */
spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
/* fill in report manufacture */
if (mpt2sas_port->remote_identify.device_type ==
MPI2_SAS_DEVICE_INFO_EDGE_EXPANDER ||
mpt2sas_port->remote_identify.device_type ==
MPI2_SAS_DEVICE_INFO_FANOUT_EXPANDER)
_transport_expander_report_manufacture(ioc,
mpt2sas_port->remote_identify.sas_address,
rphy_to_expander_device(rphy));
return mpt2sas_port;
out_fail:
list_for_each_entry_safe(mpt2sas_phy, next, &mpt2sas_port->phy_list,
port_siblings)
list_del(&mpt2sas_phy->port_siblings);
kfree(mpt2sas_port);
return NULL;
}
3.4 phy的添加
创建并添加expander phy的代码流程
/**
* mpt2sas_transport_add_expander_phy - report expander phy to transport
* @ioc: per adapter object
* @mpt2sas_phy: mpt2sas per phy object
* @expander_pg1: expander page 1
* @parent_dev: parent device class object
*
* Returns 0 for success, non-zero for failure.
*/
int
mpt2sas_transport_add_expander_phy(struct MPT2SAS_ADAPTER *ioc, struct _sas_phy
*mpt2sas_phy, Mpi2ExpanderPage1_t expander_pg1, struct device *parent_dev)
{
struct sas_phy *phy;
int phy_index = mpt2sas_phy->phy_id;
INIT_LIST_HEAD(&mpt2sas_phy->port_siblings);
phy = sas_phy_alloc(parent_dev, phy_index); /* 博主：kmalloc申请sas_phy */
if (!phy) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
if ((_transport_set_identify(ioc, mpt2sas_phy->handle,
&mpt2sas_phy->identify))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
phy->identify = mpt2sas_phy->identify;
mpt2sas_phy->attached_handle =
le16_to_cpu(expander_pg1.AttachedDevHandle);
if (mpt2sas_phy->attached_handle)
_transport_set_identify(ioc, mpt2sas_phy->attached_handle,
&mpt2sas_phy->remote_identify);
phy->identify.phy_identifier = mpt2sas_phy->phy_id;
phy->negotiated_linkrate = _transport_convert_phy_link_rate(
expander_pg1.NegotiatedLinkRate &
MPI2_SAS_NEG_LINK_RATE_MASK_PHYSICAL);
phy->minimum_linkrate_hw = _transport_convert_phy_link_rate(
expander_pg1.HwLinkRate & MPI2_SAS_HWRATE_MIN_RATE_MASK);
phy->maximum_linkrate_hw = _transport_convert_phy_link_rate(
expander_pg1.HwLinkRate >> 4);
phy->minimum_linkrate = _transport_convert_phy_link_rate(
expander_pg1.ProgrammedLinkRate & MPI2_SAS_PRATE_MIN_RATE_MASK);
phy->maximum_linkrate = _transport_convert_phy_link_rate(
expander_pg1.ProgrammedLinkRate >> 4);
/* 将phy添加到transport 层 */
if ((sas_phy_add(phy))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
sas_phy_free(phy);
return -1;
}
if ((ioc->logging_level & MPT_DEBUG_TRANSPORT))
dev_printk(KERN_INFO, &phy->dev,
"add: handle(0x%04x), sas_addr(0x%016llx)\n"
"\tattached_handle(0x%04x), sas_addr(0x%016llx)\n",
mpt2sas_phy->handle, (unsigned long long)
mpt2sas_phy->identify.sas_address,
mpt2sas_phy->attached_handle,
(unsigned long long)
mpt2sas_phy->remote_identify.sas_address);
mpt2sas_phy->phy = phy; /* 底层phy和中层建立连接*/
return 0;
}
3.5添加end device
3.5.1 end device添加到驱动
对于设备的添加：首先是驱动级别的，从IOC中获取此设备的基本信息，比如sas地址，handle等填充设备。
/**
* _scsih_add_device - creating sas device object
* @ioc: per adapter object
* @handle: sas device handle
* @phy_num: phy number end device attached to
* @is_pd: is this hidden raid component
*
* Creating end device object, stored in ioc->sas_device_list.
*
* Returns 0 for success, non-zero for failure.
*/
static int
_scsih_add_device(struct MPT2SAS_ADAPTER *ioc, u16 handle, u8 phy_num, u8 is_pd)
{
Mpi2ConfigReply_t mpi_reply;
Mpi2SasDevicePage0_t sas_device_pg0;
Mpi2SasEnclosurePage0_t enclosure_pg0;
struct _sas_device *sas_device;
u32 ioc_status;
__le64 sas_address;
u32 device_info;
unsigned long flags;
if ((mpt2sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
MPI2_IOCSTATUS_MASK;
if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
/* check if device is present */
if (!(le16_to_cpu(sas_device_pg0.Flags) &
MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
printk(MPT2SAS_ERR_FMT "Flags = 0x%04x\n",
ioc->name, le16_to_cpu(sas_device_pg0.Flags));
return -1;
}
/* check if there were any issus with discovery */
if (sas_device_pg0.AccessStatus ==
MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
printk(MPT2SAS_ERR_FMT "AccessStatus = 0x%02x\n",
ioc->name, sas_device_pg0.AccessStatus);
return -1;
}
/* check if this is end device */
device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
if (!(_scsih_is_end_device(device_info))) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
/* 首先查找target 设备，如果设备已经存在，则unblocked */
spin_lock_irqsave(&ioc->sas_device_lock, flags);
sas_device = mpt2sas_scsih_sas_device_find_by_sas_address(ioc,
sas_address);
spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
if (sas_device) {
_scsih_ublock_io_device(ioc, handle);
return 0;
}
/* 设备尚未在驱动中，则kmalloc一个 */
sas_device = kzalloc(sizeof(struct _sas_device),
GFP_KERNEL);
if (!sas_device) {
printk(MPT2SAS_ERR_FMT "failure at %s:%d/%s()!\n",
ioc->name, __FILE__, __LINE__, __func__);
return -1;
}
sas_device->handle = handle;
sas_device->parent_handle =
le16_to_cpu(sas_device_pg0.ParentDevHandle);
sas_device->enclosure_handle =
le16_to_cpu(sas_device_pg0.EnclosureHandle);
sas_device->slot =
le16_to_cpu(sas_device_pg0.Slot);
sas_device->device_info = device_info;
sas_device->sas_address = sas_address;
sas_device->hidden_raid_component = is_pd;
/* get enclosure_logical_id */
if (sas_device->enclosure_handle && !(mpt2sas_config_get_enclosure_pg0(
ioc, &mpi_reply, &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
sas_device->enclosure_handle)))
sas_device->enclosure_logical_id =
le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
/* get device name */
sas_device->device_name = le64_to_cpu(sas_device_pg0.DeviceName);
/* 添加 */
if (ioc->wait_for_port_enable_to_complete)
_scsih_sas_device_init_add(ioc, sas_device);
else
_scsih_sas_device_add(ioc, sas_device);
return 0;
}
3.5.2 end device 添加到transport
设备添加到transport 中，最终调用的接口和添加expander是一样的，即把和新插入磁盘相连的expander的phy组成一个port添加到transport，并将设备本身添加到transport中
/**
* _scsih_sas_device_add - insert sas_device to the list.
* @ioc: per adapter object
* @sas_device: the sas_device object
* Context: This function will acquire ioc->sas_device_lock.
*
* Adding new object to the ioc->sas_device_list.
*/
static void
_scsih_sas_device_add(struct MPT2SAS_ADAPTER *ioc,
struct _sas_device *sas_device)
{
unsigned long flags;
u16 handle, parent_handle;
u64 sas_address;
dewtprintk(ioc, printk(MPT2SAS_DEBUG_FMT "%s: handle"
"(0x%04x), sas_addr(0x%016llx)\n", ioc->name, __func__,
sas_device->handle, (unsigned long long)sas_device->sas_address));
spin_lock_irqsave(&ioc->sas_device_lock, flags);
list_add_tail(&sas_device->list, &ioc->sas_device_list);
spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
handle = sas_device->handle;
parent_handle = sas_device->parent_handle;
sas_address = sas_device->sas_address;
if (!mpt2sas_transport_port_add(ioc, handle, parent_handle))
_scsih_sas_device_remove(ioc, sas_device);
}
3.5.3 end device添加到scsi总线中
和expander不同的地方，磁盘作为一个end device，需要upper layer的驱动，需要将磁盘注册到OS中的scsi bus中
/**
* sas_rphy_add - add a SAS remote PHY to the device hierarchy
* @rphy: The remote PHY to be added
*
* Publishes a SAS remote PHY to the rest of the system.
*/
int sas_rphy_add(struct sas_rphy *rphy)
{
struct sas_port *parent = dev_to_sas_port(rphy->dev.parent);
struct Scsi_Host *shost = dev_to_shost(parent->dev.parent);
struct sas_host_attrs *sas_host = to_sas_host_attrs(shost);
struct sas_identify *identify = &rphy->identify;
int error;
if (parent->rphy)
return -ENXIO;
parent->rphy = rphy;
error = device_add(&rphy->dev);
if (error)
return error;
transport_add_device(&rphy->dev);
transport_configure_device(&rphy->dev);
if (sas_bsg_initialize(shost, rphy))
printk("fail to a bsg device %s\n", dev_name(&rphy->dev));
mutex_lock(&sas_host->lock);
list_add_tail(&rphy->list, &sas_host->rphy_list);
if (identify->device_type == SAS_END_DEVICE &&
(identify->target_port_protocols &
(SAS_PROTOCOL_SSP|SAS_PROTOCOL_STP|SAS_PROTOCOL_SATA)))
rphy->scsi_target_id = sas_host->next_target_id++;
else if (identify->device_type == SAS_END_DEVICE)
rphy->scsi_target_id = -1;
mutex_unlock(&sas_host->lock);
if (identify->device_type == SAS_END_DEVICE &&
rphy->scsi_target_id != -1) {
scsi_scan_target(&rphy->dev, 0,
rphy->scsi_target_id, SCAN_WILD_CARD, 0);
}
return 0;
}
最终调用中层接口进行scsi_target和scsi_device的创建和添加
static void __scsi_scan_target(struct device *parent, unsigned int channel,
unsigned int id, unsigned int lun, int rescan)
{
struct Scsi_Host *shost = dev_to_shost(parent);
int bflags = 0;
int res;
struct scsi_target *starget;
if (shost->this_id == id)
/*
* Don't scan the host adapter
*/
return;
starget = scsi_alloc_target(parent, channel, id); /* 根据进来rphy->dev及target id来创建 target */
if (!starget)
return;
if (lun != SCAN_WILD_CARD) {
/*
* Scan for a specific host/chan/id/lun.
*/
scsi_probe_and_add_lun(starget, lun, NULL, NULL, rescan, NULL);
goto out_reap;
}
/*
* Scan LUN 0, if there is some response, scan further. Ideally, we
* would not configure LUN 0 until all LUNs are scanned.
*/
res = scsi_probe_and_add_lun(starget, 0, &bflags, NULL, rescan, NULL); /* 发生INQUIRY 命令获取设备信息，并将设备添加到scsi bus上，最终OS调用upper*/
if (res == SCSI_SCAN_LUN_PRESENT || res == SCSI_SCAN_TARGET_PRESENT) {
if (scsi_report_lun_scan(starget, bflags, rescan) != 0)
/*
* The REPORT LUN did not scan the target,
* do a sequential scan.
*/
scsi_sequential_lun_scan(starget, bflags,
starget->scsi_level, rescan);
}
out_reap:
/* now determine if the target has any children at all
* and if not, nuke it */
scsi_target_reap(starget);
put_device(&starget->dev);
}
4. 设备关系
经过第三章节中描述的流程后，最终在驱动层和transport分别建立了如下的数据关系。其中在蓝色线之上的为transport层；线下的为驱动层。
5. 总结
通过此文，了解一个控制器需要管理哪些设备，后续在此基础上可以进一步学习掌握驱动及内核存储上层部分的操作流程。
参考资料：
1. 内核模块接口说明 https://www.kernel.org/doc/htmldocs/scsi/index.html