#初始测试的方法及环境
docker network create elastic
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.15.2
docker run --name es01-test --net elastic -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.15.2
docker pull docker.elastic.co/kibana/kibana:7.15.2
docker run --name kib01-test --net elastic -p 192.168.100.198:5601:5601 -e "ELASTICSEARCH_HOSTS=http://es01-test:9200" docker.elastic.co/kibana/kibana:7.15.2
docker stop es01-test
docker stop kib01-test
docker network rm elastic
docker rm es01-test
docker rm kib01-test
[root@localhost middleware]#
[root@localhost middleware]# docker network ls
NETWORK ID NAME DRIVER SCOPE
de02d40dda76 bridge bridge local
2ac66367a612 elastic bridge local
ab2a5d8c1a18 es_default bridge local
f8e93156a04a es_esnet bridge local
f97c1077f93e host host local
b4f27f924d6f middleware_default bridge local
65c203f6cf73 none null local
e28ac206dd24 run_default bridge local
#利用canal同步mysql数据库到elasticsearch
[root@localhost _data]# cat mysql.cnf
[mysql]
#设置mysql客户端默认字符集
default-character-set=utf8
socket=/var/lib/mysql/mysql.sock
[mysqld]
log-bin=mysql-bin #开启
binlog-format=ROW #选择row模式
mysql -uroot -proot
# 创建账号
CREATE USER canal IDENTIFIED BY 'canal';
# 授予权限
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
-- GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%' ;
# 刷新并应用
FLUSH PRIVILEGES;
docker pull canal/canal-server
#启动镜像
docker run --name canal -d canal/canal-server
#找到文件位置后 exit退出容器 将容器内部文件copy到外部
[root@localhost _data]# cd /data/canal/
[root@localhost canal]# ls
[root@localhost canal]# docker cp canal:/home/admin/canal-server/conf/example/instance.properties .
[root@localhost canal]# docker cp canal:/home/admin/canal-server/conf/canal.properties .
#关闭容器
docker stop canal
#移除容器
docker rm canal
docker run --name canal-adapter -p 8081:8081 -v /data/canal-adapter/conf:/opt/canal-adapter/conf -d slpcat/canal-adapter:v1.1.5
#复制canal-adapter初始配置文件
cd /data/canal-adapter/
docker cp canal-adapter:/opt/canal-adapter/conf .
#关闭容器
docker stop canal-adapter
#移除容器
docker rm canal-adapter
#修改刚刚复制的instance文件
vi /data/canal/instance.properties
# 找到数据库信息,按需更改
canal.instance.master.address=内网IP:3306
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
#按照需要同步的表选择
canal.instance.filter.regex=db.table
#创建canal-adapter具名挂载目录
mkdir -p /data/canal-adapter/conf
touch application.yml
mkdir -p /data/canal-adapter/conf/es7
cd es7
touch test.yml
application.yml
server:
port: 8081
spring:
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
default-property-inclusion: non_null
canal.conf:
mode: tcp
flatMessage: true
zookeeperHosts:
syncBatchSize: 1000
retries: 0
timeout:
accessKey:
secretKey:
consumerProperties:
canal.tcp.server.host: canal-server地址:11111
canal.tcp.batch.size: 500
canal.tcp.username:
canal.tcp.password:
srcDataSources:
defaultDS:
url: jdbc:mysql://数据库IP:数据库端口/数据库名?useUnicode=true
username: db_username
password: db_password
canalAdapters:
- instance: example # canal instance Name or mq topic name
groups:
- groupId: g1
outerAdapters:
- name: logger
- name: es7 # 该版本发现只能是es7/es6
hosts: esip地址:9300 # 127.0.0.1:9200 for rest mode
properties:
mode: transport # transport # or rest
#test.yml文件
#文件内容要对应数据库和esindex
dataSourceKey: defaultDS
destination: example
groupId: g1
esMapping:
_index: test
_id: _id
_type: _doc
upsert: true
sql: "SELECT a.id _id from t_test a "
etlCondition: "where a.>={}"
commitBatch: 3000
#编写容器编排文件
[root@localhost middleware]# cat docker-compose.yml
services:
canal:
container_name: canal
image: canal/canal-server
network_mode: host
volumes:
- /data/canal/canal.properties:/home/admin/canal-server/conf/canal.properties:rw
- /data/canal/instance.properties:/home/admin/canal-server/conf/example/instance.properties:rw
canal-adapter:
container_name: canal-adapter
image: slpcat/canal-adapter:v1.1.5
network_mode: host
volumes:
- /data/canal-adapter/conf:/opt/canal-adapter/conf:rw
elasticsearch:
container_name: elasticsearch
environment:
discovery.type: single-node
image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2
network_mode: host
volumes:
- /etc/localtime:/etc/localtime:ro
- /data/elasticsearch/logs:/user/share/elasticsearch/logs:rw
- /data/elasticsearch/config:/usr/share/elasticsearch/config:rw
- /data/elasticsearch/data:/usr/share/elasticsearch/data:rw
kinaba:
container_name: kibana
depends_on:
elasticsearch:
condition: service_started
environment:
ELASTICSEARCH_HOSTS: http://内网IP:9200
image: docker.elastic.co/kibana/kibana:7.15.2
network_mode: host
volumes:
- /etc/localtime:/etc/localtime:ro
- /data/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:rw
version: '3'
#中间件的路径
[root@localhost middleware]# ls /data/
canal canal-adapter elasticsearch kibana
[root@localhost middleware]# ls /data/canal
canal.properties instance.properties
[root@localhost middleware]# ls /data/canal-adapter/
conf
[root@localhost middleware]# ls /data/canal-adapter/conf/
application.yml es7
[root@localhost middleware]# ls /data/canal-adapter/conf/es7
test.yml
[root@localhost middleware]# ls /data/elasticsearch/
config/ data/ logs/
[root@localhost middleware]# ls /data/elasticsearch/
config/ data/ logs/
[root@localhost middleware]# ls /data/elasticsearch/
config data logs
[root@localhost middleware]# ls /data/elasticsearch/config/
elasticsearch.keystore elasticsearch.yml jvm.options jvm.options.d log4j2.file.properties log4j2.properties role_mapping.yml roles.yml users users_roles
[root@localhost middleware]# ls /data/kibana/config/
kibana.yml
#elasticsearch配置说明
cat /data/elasticsearch/config/elasticsearch.yml
cluster.name: "docker-cluster"
network.host: 0.0.0.0
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
#kibana配置文件
ls /data/kibana/config/
kibana.yml
#kibana配置说明
cat /data/kibana/config/kibana.yml
server.name: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://内网IP:9200" ]
server.publicBaseUrl: "http://内网IP:5601"
monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: "账号"
elasticsearch.password: "密码"
i18n.locale: "zh-CN"
#启动容器
docker-compose up -d
#设置账号密码
docker exec -it elasticsearch /bin/bash -c 'cd /bin'
bash-4.4# elasticsearch-setup-passwords interactive
Initiating the setup of passwords for reserved users elastic,apm_system,kibana,kibana_system,logstash_system,beats_system,remote_monitoring_user.
You will be prompted to enter passwords as the process progresses.
Please confirm that you would like to continue [y/N]y
Enter password for [elastic]:
Reenter password for [elastic]:
Passwords do not match.
Try again.
Enter password for [elastic]:
Reenter password for [elastic]:
Enter password for [apm_system]:
Reenter password for [apm_system]:
Enter password for [kibana_system]:
Reenter password for [kibana_system]:
Enter password for [logstash_system]:
Reenter password for [logstash_system]:
Enter password for [beats_system]:
Reenter password for [beats_system]:
Enter password for [remote_monitoring_user]:
Reenter password for [remote_monitoring_user]:
Changed password for user [apm_system]
Changed password for user [kibana_system]
Changed password for user [kibana]
Changed password for user [logstash_system]
Changed password for user [beats_system]
Changed password for user [remote_monitoring_user]
Changed password for user [elastic]
bash-4.4# exit
#修改配置后重启加载
docker-compose restart
#检查
[root@localhost middleware]# docker-compose ps
Name Command State Ports
--------------------------------------------------------------
canal /alidata/bin/main.sh /home ... Up
canal-adapter sh -c ./bin/startup.sh Up
elasticsearch /bin/tini -- /usr/local/bi ... Up
kibana /bin/tini -- /usr/local/bi ... Up
[root@localhost middleware]# ss -tunlp|grep 9200
tcp LISTEN 0 128 [::]:9200 [::]:* users:(("java",pid=141657,fd=307))
[root@localhost middleware]# ss -tunlp|grep 9300
tcp LISTEN 0 128 [::]:9300 [::]:* users:(("java",pid=141657,fd=305))
[root@localhost middleware]# ss -tunlp|grep 11112
tcp LISTEN 0 3 *:11112 *:* users:(("java",pid=142356,fd=71))
[root@localhost middleware]# ss -tunlp|grep 11111
tcp LISTEN 0 50 *:11111 *:* users:(("java",pid=142356,fd=87))
[root@localhost middleware]# ss -tunlp|grep 11110
tcp LISTEN 0 50 *:11110 *:* users:(("java",pid=142356,fd=91))
#访问