由于apache 官网的 docker image 是单点，要实现集群方式部署。
rocketmq 分为 nameserver 和 broker ， 对于之间调用频繁的服务，会增加网络压力， 所以 考虑又网络通信 变成 进程间通信， 那么1个pod 中包含两个container。
关于rocketmq 下载编译， https://www.cnblogs.com/fengjian2016/p/10150394.html
拿到 编译后的 apache-rocketmq , 需要修改一些配置， 例如：
. rm -rf 删除 rocketmq/bin/*.cmd
2 .把runserver.sh HEAP_OPTS 变成环境变量， 把runbroker.sh HEAP_OPTS 变成环境变量
3. 指定消息存储路径
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store/commitlog
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/logs
4. 指定日志输出路径
/data/rocketmq/conf/ 中的xml文件
sed -i 's#${user.home}#/data/rocketmq/logs#g' *.xml
5. 删除多余配置文件 2m-2s 2master 等等， 启动的时候k8s command 直接指向文件
/data/rocketmq/conf/ 目录下新建 broker-a.conf broker-b.conf 2个文件 6. 配置修改
(1) rocketmqHome 修改
(2) namesrvAddr 修改，强调 通一个pod 必须写localhost 或者 127.0.0.1 要不然调不通
[root@harbor conf]# cat broker-a.conf
rocketmqHome=/data/rocketmq
namesrvAddr=localhost:;rocketmq2:
brokerName=broker-a
brokerClusterName=DefaultCluster
brokerId=
brokerPermission=
#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数
defaultTopicQueueNums=
#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭
autoCreateTopicEnable=true
clusterTopicEnable=true
brokerTopicEnable=true
#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭
autoCreateSubscriptionGroup=true
sendMessageThreadPoolNums=
pullMessageThreadPoolNums=
queryMessageThreadPoolNums=
adminBrokerThreadPoolNums=
clientManageThreadPoolNums=
consumerManageThreadPoolNums=
heartbeatThreadPoolNums=
endTransactionThreadPoolNums=
flushConsumerOffsetInterval=
flushConsumerOffsetHistoryInterval=
rejectTransactionMessage=false
fetchNamesrvAddrByAddressServer=false
sendThreadPoolQueueCapacity=
pullThreadPoolQueueCapacity=
queryThreadPoolQueueCapacity=
clientManagerThreadPoolQueueCapacity=
consumerManagerThreadPoolQueueCapacity=
heartbeatThreadPoolQueueCapacity=
endTransactionPoolQueueCapacity=
filterServerNums=
longPollingEnable=true
shortPollingTimeMills=
notifyConsumerIdsChangedEnable=true
highSpeedMode=false
commercialEnable=true
commercialTimerCount=
commercialTransCount=
commercialBigCount=
commercialBaseCount=
transferMsgByHeap=true
maxDelayTime=
regionId=DefaultRegion
registerBrokerTimeoutMills=
slaveReadEnable=false
disableConsumeIfConsumerReadSlowly=false
consumerFallbehindThreshold=
brokerFastFailureEnable=true
waitTimeMillsInSendQueue=
waitTimeMillsInPullQueue=
waitTimeMillsInHeartbeatQueue=
waitTimeMillsInTransactionQueue=
startAcceptSendRequestTimeStamp=
traceOn=true
enableCalcFilterBitMap=false
expectConsumerNumUseFilter=
maxErrorRateOfBloomFilter=
filterDataCleanTimeSpan=
filterSupportRetry=false
enablePropertyFilter=false
compressedRegister=false
forceRegister=true
registerNameServerPeriod=
transactionTimeOut=
transactionCheckMax=
transactionCheckInterval=
#Broker 对外服务的监听端口
listenPort=
serverWorkerThreads=
serverCallbackExecutorThreads=
serverSelectorThreads=
serverOnewaySemaphoreValue=
serverAsyncSemaphoreValue=
serverChannelMaxIdleTimeSeconds=
serverSocketSndBufSize=
serverSocketRcvBufSize=
serverPooledByteBufAllocatorEnable=true
useEpollNativeSelector=false
clientWorkerThreads=
clientCallbackExecutorThreads=
clientOnewaySemaphoreValue=
clientAsyncSemaphoreValue=
connectTimeoutMillis=
channelNotActiveInterval=
clientChannelMaxIdleTimeSeconds=
clientSocketSndBufSize=
clientSocketRcvBufSize=
clientPooledByteBufAllocatorEnable=false
clientCloseSocketIfTimeout=false
useTLS=false
storePathRootDir=/data/rocketmq/store
storePathCommitLog=/root/rocketmq/store/commitlog
#commitLog每个文件的大小默认1G
mapedFileSizeCommitLog=
#ConsumeQueue每个文件默认存60W条，根据业务情况调整,
mapedFileSizeConsumeQueue=
enableConsumeQueueExt=false
mappedFileSizeConsumeQueueExt=
bitMapLengthConsumeQueueExt=
flushIntervalCommitLog=
commitIntervalCommitLog=
useReentrantLockWhenPutMessage=false
flushCommitLogTimed=false
flushIntervalConsumeQueue=
cleanResourceInterval=
deleteCommitLogFilesInterval=
deleteConsumeQueueFilesInterval=
destroyMapedFileIntervalForcibly=
redeleteHangedFileInterval=
#删除文件时间点，默认凌晨 2点
deleteWhen=
#检测物理文件磁盘空间
diskMaxUsedSpaceRatio=
#文件保留时间，默认 小时
fileReservedTime=
putMsgIndexHightWater=
#限制的消息大小
maxMessageSize=
checkCRCOnRecover=true
flushCommitLogLeastPages=
commitCommitLogLeastPages=
flushLeastPagesWhenWarmMapedFile=
flushConsumeQueueLeastPages=
flushCommitLogThoroughInterval=
commitCommitLogThoroughInterval=
flushConsumeQueueThoroughInterval=
maxTransferBytesOnMessageInMemory=
maxTransferCountOnMessageInMemory=
maxTransferBytesOnMessageInDisk=
maxTransferCountOnMessageInDisk=
accessMessageInMemoryMaxRatio=
messageIndexEnable=true
maxHashSlotNum=
maxIndexNum=
maxMsgsNumBatch=
messageIndexSafe=false
haListenPort=
haSendHeartbeatInterval=
haHousekeepingInterval=
haTransferBatchSize=
haMasterAddress=
haSlaveFallbehindMax=
brokerRole=ASYNC_MASTER
flushDiskType=ASYNC_FLUSH
syncFlushTimeout=
messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
flushDelayOffsetInterval=
cleanFileForciblyEnable=true
warmMapedFileEnable=false
offsetCheckInSlave=false
debugLockEnable=false
duplicationEnable=false
diskFallRecorded=true
osPageCacheBusyTimeOutMills=
defaultQueryMaxNum=
transientStorePoolEnable=false
transientStorePoolSize=
fastFailIfNoBufferInStorePool=false
[root@harbor conf]# cat broker-b.conf
rocketmqHome=/data/rocketmq
namesrvAddr=rocketmq1:;localhost:
brokerName=broker-b
brokerClusterName=DefaultCluster
brokerId=
brokerPermission=
#在发送消息时，自动创建服务器不存在的topic，默认创建的队列数
defaultTopicQueueNums=
#是否允许 Broker 自动创建Topic，建议线下开启，线上关闭
autoCreateTopicEnable=true
clusterTopicEnable=true
brokerTopicEnable=true
#是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭
autoCreateSubscriptionGroup=true
sendMessageThreadPoolNums=
pullMessageThreadPoolNums=
queryMessageThreadPoolNums=
adminBrokerThreadPoolNums=
clientManageThreadPoolNums=
consumerManageThreadPoolNums=
heartbeatThreadPoolNums=
endTransactionThreadPoolNums=
flushConsumerOffsetInterval=
flushConsumerOffsetHistoryInterval=
rejectTransactionMessage=false
fetchNamesrvAddrByAddressServer=false
sendThreadPoolQueueCapacity=
pullThreadPoolQueueCapacity=
queryThreadPoolQueueCapacity=
clientManagerThreadPoolQueueCapacity=
consumerManagerThreadPoolQueueCapacity=
heartbeatThreadPoolQueueCapacity=
endTransactionPoolQueueCapacity=
filterServerNums=
longPollingEnable=true
shortPollingTimeMills=
notifyConsumerIdsChangedEnable=true
highSpeedMode=false
commercialEnable=true
commercialTimerCount=
commercialTransCount=
commercialBigCount=
commercialBaseCount=
transferMsgByHeap=true
maxDelayTime=
regionId=DefaultRegion
registerBrokerTimeoutMills=
slaveReadEnable=false
disableConsumeIfConsumerReadSlowly=false
consumerFallbehindThreshold=
brokerFastFailureEnable=true
waitTimeMillsInSendQueue=
waitTimeMillsInPullQueue=
waitTimeMillsInHeartbeatQueue=
waitTimeMillsInTransactionQueue=
startAcceptSendRequestTimeStamp=
traceOn=true
enableCalcFilterBitMap=false
expectConsumerNumUseFilter=
maxErrorRateOfBloomFilter=
filterDataCleanTimeSpan=
filterSupportRetry=false
enablePropertyFilter=false
compressedRegister=false
forceRegister=true
registerNameServerPeriod=
transactionTimeOut=
transactionCheckMax=
transactionCheckInterval=
#Broker 对外服务的监听端口
listenPort=
serverWorkerThreads=
serverCallbackExecutorThreads=
serverSelectorThreads=
serverOnewaySemaphoreValue=
serverAsyncSemaphoreValue=
serverChannelMaxIdleTimeSeconds=
serverSocketSndBufSize=
serverSocketRcvBufSize=
serverPooledByteBufAllocatorEnable=true
useEpollNativeSelector=false
clientWorkerThreads=
clientCallbackExecutorThreads=
clientOnewaySemaphoreValue=
clientAsyncSemaphoreValue=
connectTimeoutMillis=
channelNotActiveInterval=
clientChannelMaxIdleTimeSeconds=
clientSocketSndBufSize=
clientSocketRcvBufSize=
clientPooledByteBufAllocatorEnable=false
clientCloseSocketIfTimeout=false
useTLS=false
storePathRootDir=/data/rocketmq/store
storePathCommitLog=/root/rocketmq/store/commitlog
#commitLog每个文件的大小默认1G
mapedFileSizeCommitLog=
#ConsumeQueue每个文件默认存60W条，根据业务情况调整,
mapedFileSizeConsumeQueue=
enableConsumeQueueExt=false
mappedFileSizeConsumeQueueExt=
bitMapLengthConsumeQueueExt=
flushIntervalCommitLog=
commitIntervalCommitLog=
useReentrantLockWhenPutMessage=false
flushCommitLogTimed=false
flushIntervalConsumeQueue=
cleanResourceInterval=
deleteCommitLogFilesInterval=
deleteConsumeQueueFilesInterval=
destroyMapedFileIntervalForcibly=
redeleteHangedFileInterval=
#删除文件时间点，默认凌晨 2点
deleteWhen=
#检测物理文件磁盘空间
diskMaxUsedSpaceRatio=
#文件保留时间，默认 小时
fileReservedTime=
putMsgIndexHightWater=
#限制的消息大小
maxMessageSize=
checkCRCOnRecover=true
flushCommitLogLeastPages=
commitCommitLogLeastPages=
flushLeastPagesWhenWarmMapedFile=
flushConsumeQueueLeastPages=
flushCommitLogThoroughInterval=
commitCommitLogThoroughInterval=
flushConsumeQueueThoroughInterval=
maxTransferBytesOnMessageInMemory=
maxTransferCountOnMessageInMemory=
maxTransferBytesOnMessageInDisk=
maxTransferCountOnMessageInDisk=
accessMessageInMemoryMaxRatio=
messageIndexEnable=true
maxHashSlotNum=
maxIndexNum=
maxMsgsNumBatch=
messageIndexSafe=false
haListenPort=
haSendHeartbeatInterval=
haHousekeepingInterval=
haTransferBatchSize=
haMasterAddress=
haSlaveFallbehindMax=
brokerRole=ASYNC_MASTER
flushDiskType=ASYNC_FLUSH
syncFlushTimeout=
messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
flushDelayOffsetInterval=
cleanFileForciblyEnable=true
warmMapedFileEnable=false
offsetCheckInSlave=false
debugLockEnable=false
duplicationEnable=false
diskFallRecorded=true
osPageCacheBusyTimeOutMills=
defaultQueryMaxNum=
transientStorePoolEnable=false
transientStorePoolSize=
fastFailIfNoBufferInStorePool=false
broker Dockerfile
[root@harbor rocketmq-broker]# cat Dockerfile
FROM centos MAINTAINER fengjian <fengjian@senyint.com> ENV TZ "Asia/Shanghai"
ENV TERM xterm
ENV MALLOC_ARENA_MAX=
ENV JAVA_HOME /data/jdk
ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV CATALINA_HOME /data/tomcat
ENV CATALINA_BASE /data/tomcat
ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin
ENV ROCKETMQ_HOME /data/rocketmq RUN rm /etc/yum.repos.d/* -rf
ADD Centos-7.repo /etc/yum.repos.d/
ADD epel.repo /etc/yum.repos.d/
RUN yum clean all
RUN yum -y install net-tools bind-utils sysstat ADD localtime /etc/
ADD jdk.tar.gz /data
ADD profile /etc
ADD sysctl.conf /etc
ADD 20-nproc.conf /etc/security/limits.d/
ADD limits.conf /etc/security/ ADD rocketmq.tar.gz /data/
WORKDIR ${ROCKETMQ_HOME}/bin EXPOSE 10909 10911
nameserver Dockerfile
FROM centos MAINTAINER fengjian <fengjian@senyint.com> ENV TZ "Asia/Shanghai"
ENV TERM xterm
ENV MALLOC_ARENA_MAX=
ENV JAVA_HOME /data/jdk
ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV CATALINA_HOME /data/tomcat
ENV CATALINA_BASE /data/tomcat
ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin
ENV ROCKETMQ_HOME /data/rocketmq RUN rm /etc/yum.repos.d/* -rf
ADD Centos-7.repo /etc/yum.repos.d/
ADD epel.repo /etc/yum.repos.d/
RUN yum clean all
RUN yum -y install net-tools bind-utils sysstat ADD localtime /etc/
ADD jdk.tar.gz /data
ADD profile /etc
ADD sysctl.conf /etc
ADD 20-nproc.conf /etc/security/limits.d/
ADD limits.conf /etc/security/ ADD rocketmq.tar.gz /data/
WORKDIR ${ROCKETMQ_HOME}/bin EXPOSE 9876 ENTRYPOINT ["sh","mqnamesrv"]
cat ceph-rocketmq1-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rocketmq1-brokerstore-pvc
annotations:
volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
accessModes:
- ReadWriteOnce #必须写ReadWriteOnce,否则报错。
resources:
requests:
storage: 20Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rocketmq1-brokerlog-pvc
annotations:
volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
accessModes:
- ReadWriteOnce #必须写ReadWriteOnce,否则报错。
resources:
requests:
storage: 20Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: rocketmq1-namesrvlog-pvc
annotations:
volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq
spec:
accessModes:
- ReadWriteOnce #必须写ReadWriteOnce,否则报错。
resources:
requests:
storage: 20Gi
rocketmq1 yaml文件
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: rocketmq1
spec:
replicas:
template:
metadata:
labels:
app: rocketmq1
spec:
containers:
- name: namesrv
image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3
imagePullPolicy: Always
ports:
- containerPort:
env:
- name: ROCKETMQ_HEAP_OPTS
value : "-Xms1g -Xmx1g -Xmn512m"
volumeMounts:
- name: rocketmq1namesrvlog
mountPath: /data/rocketmq/logs
- name: broker
image: 192.168.200.10/fengjian/rocketmq-broker:4.3
imagePullPolicy: Always
ports:
- name: brockerserver
containerPort:
- name: brockerslave
containerPort:
command: ["/data/rocketmq/bin/mqbroker"]
args: ["-c", "/data/rocketmq/conf/broker-a.conf"]
env:
- name: ROCKETMQ_HEAP_OPTS
value : "-Xms2g -Xmx2g -Xmn1g"
volumeMounts:
- name: brokerlogs
mountPath: /data/rocketmq/logs
- name: brokerstore
mountPath: /data/rocketmq/store
volumes:
- name: rocketmq1namesrvlog
persistentVolumeClaim:
claimName: rocketmq1-namesrvlog-pvc
- name: brokerlogs
persistentVolumeClaim:
claimName: rocketmq1-brokerlog-pvc
- name: brokerstore
persistentVolumeClaim:
claimName: rocketmq1-brokerstore-pvc ---
apiVersion: v1
kind: Service
metadata:
name: rocketmq1
spec:
ports:
- name: brokerslave
port:
protocol: TCP
targetPort:
- name: brokerserver
port:
protocol: TCP
targetPort:
- name: namesrvport
port:
protocol: TCP
targetPort:
selector:
app: rocketmq1
还要提前定义pvc 卷
vim rocketmq2.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: rocketmq1
spec:
replicas:
template:
metadata:
labels:
app: rocketmq1
spec:
containers:
- name: namesrv
image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3
imagePullPolicy: Always
ports:
- containerPort:
env:
- name: ROCKETMQ_HEAP_OPTS
value : "-Xms1g -Xmx1g -Xmn512m"
volumeMounts:
- name: rocketmq1namesrvlog
mountPath: /data/rocketmq/logs
- name: broker
image: 192.168.200.10/fengjian/rocketmq-broker:4.3
imagePullPolicy: Always
ports:
- name: brockerserver
containerPort:
- name: brockerslave
containerPort:
command: ["/data/rocketmq/bin/mqbroker"]
args: ["-c", "/data/rocketmq/conf/broker-b.conf"]
env:
- name: ROCKETMQ_HEAP_OPTS
value : "-Xms2g -Xmx2g -Xmn1g"
volumeMounts:
- name: brokerlogs
mountPath: /data/rocketmq/logs
- name: brokerstore
mountPath: /data/rocketmq/store
volumes:
- name: rocketmq1namesrvlog
persistentVolumeClaim:
claimName: rocketmq1-namesrvlog-pvc
- name: brokerlogs
persistentVolumeClaim:
claimName: rocketmq1-brokerlog-pvc
- name: brokerstore
persistentVolumeClaim:
claimName: rocketmq1-brokerstore-pvc ---
apiVersion: v1
kind: Service
metadata:
name: rocketmq1
spec:
ports:
- name: brokerslave
port:
protocol: TCP
targetPort:
- name: brokerserver
port:
protocol: TCP
targetPort:
- name: namesrvport
port:
protocol: TCP
targetPort:
selector:
app: rocketmq1