Lenovo DE6000H实现IB网络访问操作手册
- HPC存储
- 12小时前
- 13热度
- 0评论
文章标签： 网络 服务器 linux
一、LENOVO DE6000H 存储配置:
网线接入P1口：https://192.168.128.101
用户名：admin 密码：xxx
2.创建卷组：raid6(8+1=1) 名称data
同样的方法创建卷组2 名称data2
3. 创建全局热备盘，防止硬盘损坏，没有及时发现。
4.创建主机：名称IO,操作系统类型linux DM MP,主机端口在IO（rhel7.4主机上查看）具体命令：cat /etc/iscsi/initiatorname.iscsi
5.配置iser over infiniband 端口IP
A控c0 端口192.168.168.111
B控c0 端口192.168.168.111
6.从刚刚创建的卷组data,data2中，创建卷 名称io 容量23TB 。名称io1 容量20TB
7. 将创建的卷，与创建的主机进行映射：
以上是存储上的配置。
#########################################################
二、下面进行LINUX主机配置：做此操作时，将存储ISCSI转成FC端口》
##################
1.配置子网管理器
%%%解决infiniband存储端口灯不亮的问题：（ib交换机侧红灯，存储侧ib灯不亮）
systemctl start opensm
/sbin/chkconfig opensmd on
################
ibstat -p //查看ib卡GUID 因为是双端口卡所以是2个
0x98039b0300c40ea2
0x98039b0300c40ea3
vi /etc/rc.d/after.local
opensm -B -g 0x98039b0300c40ea2 -p P0 -f /var/log/opensm-ib0.log
opensm -B -g 0x98039b0300c40ea3 -p P1 -f /var/log/opensm-ib1.log
#######################################################################################################
2.
rhel7.4 主机安装多路径软件
yum install device-mapper-multipath //多路径软件配置安装
多路缉径配置：
vi multipath.conf
devices{
device {
vendor "LENOVO"
product "DE_Series"
product_blacklist "Universal Xport"
path_grouping_policy group_by_prio
features "2 pg_init_retries 50"
path_checker rdac
path_selector "round-robin 0"
hardware_handler "1 rdac"
prio rdac
failback immediate
rr_weight priorities
no_path_retry 30
}
}
########
启动多路径级LINUX主机开机自动引导
systemctl start multipathd
systemctl enable multipathd
systemctl is-enabled multipathd
dm-mp写入内核
dracut --force --add multipath
###########################################################################linux
3.modprobe ib_iser //将 ib_iser 写入LINUX内核
4.lsmod | grep iser //显示
[root@io ~]# lsmod | grep iser
ib_iser 52337 6
libiscsi 57233 3 libiscsi_tcp,iscsi_tcp,ib_iser
scsi_transport_iscsi 99909 4 iscsi_tcp,ib_iser,libiscsi
rdma_cm 55229 2 ib_iser,rdma_ucm
ib_core 294802 11 rdma_cm,ib_cm,iw_cm,mlx4_ib,mlx5_ib,ib_ucm,ib_iser,ib_umad,ib_uverbs,rdma_ucm,ib_ipoib
mlx_compat 29658 16 rdma_cm,ib_cm,iw_cm,mlx4_en,mlx4_ib,mlx5_ib,mlx5_fpga_tools,ib_ucm,ib_core,ib_iser,ib_umad,ib_uverbs,mlx4_core,mlx5_core,rdma_ucm,ib_ipoib
##################################
###########################################################################
5.安装ISCSI启动器软件
yum install iscsi*
cat /etc/iscsi/initiatorname.iscsi //写入存储的主机
InitiatorName=iqn.1994-05.com.redhat:a6919639ed8
iscsiadm -m iface -I iser > /var/lib/iscsi/ifaces/iface-ib0
iscsiadm -m iface -I iser > /var/lib/iscsi/ifaces/iface-ib1
########
vi /var/lib/iscsi/ifaces/iface-ib0
iface.net_ifacename = ib0
iface.initiatorname = iqn.1994-05.com.redhat:a6919639ed8
########
vi /var/lib/iscsi/ifaces/iface-ib1
iface.net_ifacename = ib0
iface.initiatorname = iqn.1994-05.com.redhat:a6919639ed8
###########################################################################
6.
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -I iface-ib0 -p 192.168.168.111 -l -o new
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -I iface-ib0 -p 192.168.168.112 -l -o new
####登录存储设置：
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.112:3260 -l
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.111:3260 -l
###########自动登录设置：
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.111 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.112 --op update -n node.startup -v automatic
##########################################################################
7.查看会话
iscsiadm -m node -o show
iscsiadm -m session
iser: [1] 192.168.168.111:3260,1 iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 (non-flash)
iser: [2] 192.168.168.112:3260,2 iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 (non-flash)
#############################
8.加入到LINUX自动启动，不能加入到/etc/fstab会引起，redhat不能自动启动
因为fstab启动时，iscsi服务还没有完成加载-----切记！！！！
vi /etc/rc.d/rc.local
su - root -c "iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.112:3260 -l" sleep 10
su - root -c "iscsiadm -m node -T iqn.2002-09.com.lenovo:thinksystem.600a098000f87cf4000000005cd978f0 -p 192.168.168.111:3260 -l" sleep 10
su - root -c "mount /dev/mapper/3600a098000f87888000002c35cdaf30c1 /data"
9.查看多路径设置：
[root@io ~]# multipath -ll
mpatha (3600a098000f87888000002c35cdaf30c) dm-0 LENOVO ,DE_Series
size=23T features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=10 status=active
| `- 25:0:0:1 sdc 8:32 active ready running
`-+- policy='round-robin 0' prio=8 status=enabled
`- 24:0:0:1 sdb 8:16 active ready running
8.fio测试：进行读硬盘测试
fio --rw=read --bs=128k --numjobs=1 --iodepth=4 --size="200m" --loops=1 --ioengine=libaio --direct=1 --invalidate=1 --fsync_on_close=1 --randrepeat=1 --norandommap --exitall --name "test1" --filename="/dev/mapper/mpatha" --output=/tmp/fio_verify.sh-22334.log --time_based --runtime=305
Jobs: 1 (fbs: 1 (f=1)=1): [R(1)][1.0%][r=5825MiB/s,w=0KiB/s][r=46.6k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][1.3%][r=5844MiB/s,w=0KiB/s][r=46.7k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][1.6%][r=5867MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][2.0%][r=5869MiB/s,w=0KiB/s][r=46.0k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][2.3%][r=5862MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJo: [R(1)][2.6%][r=5859MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][3.0%][r=5864MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][3.3%][r=5857MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][3.6%][r=5856MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][3.9%][r=5796MiB/s,w=0KiB/s][r=46.4k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][4.3%][r=5819MiB/s,w=0KiB/s][r=46.6k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][4.6%][r=5838MiB/s,w=0KiB/s][r=46.7k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][4.9%][r=5857MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][5.2%][r=5849MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][5.6%][r=5861MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][5.9%][r=5854MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][6.2%][r=5854MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][6.6%][r=5861MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][6.9%][r=5853MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][7.2%][r=5852MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][7.5%][r=5811MiB/s,w=0KiB/s][r=46.5k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][7.9%][r=5864MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][8.2%][r=5861MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][8.5%][r=5865MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][8.9%][r=5862MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][9.2%][r=5860MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][9.5%][r=5856MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][9.8%][r=5880MiB/s,w=0KiB/s][r=47.0k,w=0 IOPS][etaJobs: 1 (f=1): [R(1)][10.2%][r=5859MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][10.5%][r=5862MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][10.8%][r=5830MiB/s,w=0KiB/s][r=46.6k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][11.1%][r=5821MiB/s,w=0KiB/s][r=46.6k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][11.5%][r=5851MiB/s,w=0KiB/s][r=46.8k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][11.8%][r=5859MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][12.2%][r=5857MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][12.5%][r=5859MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1): [R(1)][12.8%][r=5859MiB/s,w=0KiB/s][r=46.9k,w=0 IOPS][etJobs: 1 (f=1):
