createrepo /home/repo/rocky/8/baseos createrepo /home/repo/rocky/8/extras createrepo /home/repo/rocky/8/appstream createrepo /home/repo/rocky/8/epel createrepo /home/repo/rocky/8/elrepo-kernel ]# vim /etc/yum.conf (......) exclude=kernel* exclude=kmod* ]# curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | sudo bash ]# cat /etc/yum.repos.d/mariadb.repo [mariadb-main] name = MariaDB Server baseurl = https://downloads.mariadb.com/MariaDB/mariadb-10.4/yum/rhel/$releasever/$basearch gpgkey = file:///etc/pki/rpm-gpg/MariaDB-Server-GPG-KEY gpgcheck = 1 enabled = 1 module_hotfixes = 1 [mariadb-maxscale] # To use the latest stable release of MaxScale, use "latest" as the version # To use the latest beta (or stable if no current beta) release of MaxScale, use "beta" as the version name = MariaDB MaxScale baseurl = https://downloads.mariadb.com/MaxScale/2.4/centos/$releasever/$basearch gpgkey = file:///etc/pki/rpm-gpg/MariaDB-MaxScale-GPG-KEY gpgcheck = 1 enabled = 1 [mariadb-tools] name = MariaDB Tools baseurl = https://downloads.mariadb.com/Tools/rhel/$releasever/$basearch gpgkey = file:///etc/pki/rpm-gpg/MariaDB-Enterprise-GPG-KEY gpgcheck = 1 enabled = 1 ]# dnf install galera-4 MariaDB-backup MariaDB-client MariaDB-compat MariaDB-devel MariaDB-server (......) Installed: MariaDB-backup-10.4.13-1.el8.x86_64 MariaDB-client-10.4.13-1.el8.x86_64 MariaDB-compat-10.4.13-1.el8.x86_64 MariaDB-devel-10.4.13-1.el8.x86_64 MariaDB-server-10.4.13-1.el8.x86_64 galera-4-26.4.4-1.rhel8.0.el8.x86_64 perl-IO-Socket-IP-0.39-5.el8.noarch perl-IO-Socket-SSL-2.066-3.el8.noarch perl-Mozilla-CA-20160104-7.el8.noarch boost-program-options-1.66.0-6.el8.x86_64 compat-openssl10-1:1.0.2o-3.el8.x86_64 perl-DBI-1.641-3.module_el8.1.0+199+8f0a6bbd.x86_64 perl-Digest-1.17-395.el8.noarch perl-Digest-MD5-2.55-396.el8.x86_64 perl-Net-SSLeay-1.88-1.el8.x86_64 perl-URI-1.73-3.el8.noarch perl-libnet-3.11-3.el8.noarch libaio-0.3.112-1.el8.x86_64 libnsl-2.28-72.el8_1.1.x86_64 libpkgconf-1.4.2-1.el8.x86_64 lsof-4.91-2.el8.x86_64 make-1:4.2.1-9.el8.x86_64 perl-Carp-1.42-396.el8.noarch perl-Data-Dumper-2.167-399.el8.x86_64 perl-Encode-4:2.97-3.el8.x86_64 perl-Errno-1.28-416.el8.x86_64 perl-Exporter-5.72-396.el8.noarch perl-File-Path-2.15-2.el8.noarch perl-File-Temp-0.230.600-1.el8.noarch perl-Getopt-Long-1:2.50-4.el8.noarch perl-HTTP-Tiny-0.074-1.el8.noarch perl-IO-1.38-416.el8.x86_64 perl-MIME-Base64-3.15-396.el8.x86_64 perl-Math-BigInt-1:1.9998.11-7.el8.noarch perl-Math-Complex-1.59-416.el8.noarch perl-PathTools-3.74-1.el8.x86_64 perl-Pod-Escapes-1:1.07-395.el8.noarch perl-Pod-Perldoc-3.28-396.el8.noarch perl-Pod-Simple-1:3.35-395.el8.noarch perl-Pod-Usage-4:1.69-395.el8.noarch perl-Scalar-List-Utils-3:1.49-2.el8.x86_64 perl-Socket-4:2.027-3.el8.x86_64 perl-Storable-1:3.11-3.el8.x86_64 perl-Term-ANSIColor-4.06-396.el8.noarch perl-Term-Cap-1.17-395.el8.noarch perl-Text-ParseWords-3.30-395.el8.noarch perl-Text-Tabs+Wrap-2013.0523-395.el8.noarch perl-Time-Local-1:1.280-1.el8.noarch perl-Unicode-Normalize-1.25-396.el8.x86_64 perl-constant-1.33-396.el8.noarch perl-interpreter-4:5.26.3-416.el8.x86_64 perl-libs-4:5.26.3-416.el8.x86_64 perl-macros-4:5.26.3-416.el8.x86_64 perl-parent-1:0.237-1.el8.noarch perl-podlators-4.11-1.el8.noarch perl-threads-1:2.21-2.el8.x86_64 perl-threads-shared-1.58-2.el8.x86_64 pkgconf-1.4.2-1.el8.x86_64 pkgconf-m4-1.4.2-1.el8.noarch pkgconf-pkg-config-1.4.2-1.el8.x86_64 rsync-3.1.3-6.el8.x86_64 tar-2:1.30-4.el8.x86_64 MariaDB-common-10.4.13-1.el8.x86_64 ]# dnf list galera-4 MariaDB-backup MariaDB-client MariaDB-compat MariaDB-devel MariaDB-server Last metadata expiration check: 0:04:27 ago on Mon 15 Jun 2020 04:19:59 AM EDT. Installed Packages MariaDB-backup.x86_64 10.4.13-1.el8 @mariadb-main MariaDB-client.x86_64 10.4.13-1.el8 @mariadb-main MariaDB-compat.x86_64 10.4.13-1.el8 @mariadb-main MariaDB-devel.x86_64 10.4.13-1.el8 @mariadb-main MariaDB-server.x86_64 10.4.13-1.el8 @mariadb-main galera-4.x86_64 26.4.4-1.rhel8.0.el8 @mariadb-main ----------------------------------------------------- 4444 (SST , State Snapshot Transfer) 새 노드가 클러스터에 참여하면 SST를 실행해 다른 노드와 동기화 4568 (IST , Incremental State Transfer) 기준 노드와 비교할 때, 누락된 트랜잭션이 존재하면 동기화 4567 (Galera Cluster) 각 노드의 연동 상태를 체크하고, 멀티 캐스트 복제시 사용 (TCP/UDP) 3306 (MariaDB) MariaDB 클라이언트 연결 및 상태 스냅 샷 전송 ]# firewall-cmd --permanent --zone=public --add-port=30306/tcp success ]# firewall-cmd --permanent --zone=public --add-port=4567/tcp success ]# firewall-cmd --permanent --zone=public --add-port=4567/udp success ]# firewall-cmd --permanent --zone=public --add-port=4568/tcp success ]# firewall-cmd --permanent --zone=public --add-port=4444/tcp success ]# firewall-cmd --reload success ]# cat /etc/firewalld/zones/public.xml Public For use in public areas. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted. ----------------------------------------------------- VIP = 10.0.0.101 RIP = 10.0.0.102, 10.0.0.103, 10.0.0.104 ]# yum install -y *arptables Installed: iptables-arptables-1.8.2-16.el8.x86_64 ]# cat /etc/LVS_Vip/arp.sh # LVS-Galera VIP1="192.168.11.101" RIP1="192.168.11.102" arptables -A INPUT -j DROP -d $VIP1 arptables -A OUTPUT -j mangle -o ens18 -s $VIP1 --mangle-ip-s $RIP1 VIP2="10.0.0.101" RIP2="10.0.0.102" arptables -A INPUT -j DROP -d $VIP2 arptables -A OUTPUT -j mangle -o ens19 -s $VIP2 --mangle-ip-s $RIP2 ]# cd /etc/sysconfig/network-scripts/ ]# ll total 16 -rw-r--r-- 1 root root 329 Jun 16 14:35 ifcfg-ens18 -rw-r--r-- 1 root root 181 Jun 16 14:26 ifcfg-ens18:0 -rw-r--r-- 1 root root 272 Jun 16 13:48 ifcfg-ens19 -rw-r--r-- 1 root root 177 Jun 16 14:25 ifcfg-ens19:0 ]# cat ifcfg-ens18:0 TYPE=Ethernet PROXY_METHOD=none BROWSER_ONLY=no BOOTPROTO=none IPADDR=192.168.11.101 PREFIX=24 DEFROUTE=yes IPV4_FAILURE_FATAL=no IPV6INIT=no NAME=ens18:0 DEVICE=ens18:0 ONBOOT=yes ]# cat ifcfg-ens19:0 TYPE=Ethernet PROXY_METHOD=none BROWSER_ONLY=no BOOTPROTO=none IPADDR=10.0.0.101 PREFIX=24 DEFROUTE=yes IPV4_FAILURE_FATAL=no IPV6INIT=no NAME=ens19:0 DEVICE=ens19:0 ONBOOT=yes ----------------------------------------------------- ]# sysctl -a | grep ip_forward net.ipv4.ip_forward = 0 net.ipv4.ip_forward_update_priority = 1 net.ipv4.ip_forward_use_pmtu = 0 # vi /etc/sysctl.conf net.ipv4.ip_forward = 1 ]# sysctl -p net.ipv4.ip_forward = 1 ]# sysctl -a | grep ip_forward net.ipv4.ip_forward = 1 net.ipv4.ip_forward_update_priority = 1 net.ipv4.ip_forward_use_pmtu = 0 ]# vi /etc/selinux/config SELINUX=disabled ----------------------------------------------------- ]# cat my.cnf # # These groups are read by MariaDB server. # Use it for options that only the server (but not clients) should see # # See the examples of server my.cnf files in /usr/share/mysql/ # [client] port = 30306 socket = /tmp/galera.sock default-character-set=utf8 # this is read by the standalone daemon and embedded servers [server] # this is only for the mysqld standalone daemon [mysqld] port = 30306 socket = /tmp/galera.sock sql_mode=NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION #log-bin=/var/lib/mysql/bin_log/bin #innodb_data_home_dir = /var/lib/mysql/ibdata #innodb_log_group_home_dir = /Data/mysql_log/iblogs log-error=err-log.log # slow query long_query_time = 2 slow-query-log-file = slow-query.log skip-name-resolve # # * Galera-related settings # [galera] # Mandatory settings wsrep_cluster_name=Galera-DFIS wsrep_on=ON wsrep_provider=/usr/lib64/galera-4/libgalera_smm.so #wsrep_cluster_address=gcomm:// #wsrep_cluster_address=gcomm://10.0.0.103 wsrep_cluster_address=gcomm://10.0.0.111,10.0.0.112,10.0.0.113 wsrep_node_address=10.0.0.111 wsrep_node_name=db1.galera #wsrep_sst_receive_address=10.0.0.103:4444 #wsrep_sst_method=mysqldump wsrep_sst_method=rsync #wsrep_sst_method=mariabackup #wsrep_sst_auth=root:rksE~~ wsrep_max_ws_rows=0 wsrep_max_ws_size=2G #wsrep_provider_options="gcache.recover=yes" #wsrep_provider_options=gcache.size=128M #wsrep_log_conflicts=ON #wsrep_provider_options="cert.log_conflicts=ON" #wsrep_debug=ON #wsrep-forced-binlog-format=ROW #wsrep_slave_threads=4 binlog_format=ROW #innodb_autoinc_lock_mode=2 #innodb_flush_log_at_trx_commit=1 default_storage_engine=InnoDB # # Allow server to accept connections on all interfaces. # bind-address=0.0.0.0 # # Optional setting #wsrep_slave_threads=1 #innodb_flush_log_at_trx_commit=0 # this is only for embedded server [embedded] # This group is only read by MariaDB servers, not by MySQL. # If you use the same .cnf file for MySQL and MariaDB, # you can put MariaDB-only options here [mariadb] # This group is only read by MariaDB-10.4 servers. # If you use the same .cnf file for MariaDB of different versions, # use this group for options that older servers don't understand [mariadb-10.4] ----------------------------------------------------- 첫번째 노드 시작시 ]# galera_new_cluster ]# systemctl status mariadb.service ● mariadb.service - MariaDB 10.4.13 database server Loaded: loaded (/usr/lib/systemd/system/mariadb.service; disabled; vendor preset: disabled) Drop-In: /etc/systemd/system/mariadb.service.d └─migrated-from-my.cnf-settings.conf Active: active (running) since Tue 2020-06-16 16:48:55 KST; 5min ago Docs: man:mysqld(8) https://mariadb.com/kb/en/library/systemd/ Process: 2451 ExecStartPost=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited,> Process: 2321 ExecStartPre=/bin/sh -c [ ! -e /usr/bin/galera_recovery ] && VAR= || VAR=`cd /usr/bin/> Process: 2319 ExecStartPre=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited, > Main PID: 2415 (mysqld) Status: "Taking your SQL requests now..." Tasks: 36 (limit: 26213) Memory: 103.0M CGroup: /system.slice/mariadb.service └─2415 /usr/sbin/mysqld --wsrep-new-cluster --wsrep_start_position=93b28702-afa1-11ea-b0fe-13> Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 0 [Note] Added new Master_info '' to hash table Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 0 [Note] /usr/sbin/mysqld: ready for connection> Jun 16 16:48:55 DB1.C8 mysqld[2415]: Version: '10.4.13-MariaDB' socket: '/tmp/galera.sock' port: 30306> Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: Lowest cert indnex boundary for> Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: Min available from gcache for C> Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: Server db1.galera synced with g> Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: Server status change joined -> > Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: Synchronized with group, ready > Jun 16 16:48:55 DB1.C8 mysqld[2415]: 2020-06-16 16:48:55 2 [Note] WSREP: wsrep_notify_cmd is not defined> Jun 16 16:48:55 DB1.C8 systemd[1]: Started MariaDB 10.4.13 database server. lines 1-27/27 (END) ]# mysqladmin -S /tmp/galera.sock -u root password '비밀번호' ]# mysql -uroot -p Enter password: Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 13 Server version: 10.4.13-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> show status like '%wsrep%'; +-------------------------------+------------------------------------------------------------------+ | Variable_name | Value | +-------------------------------+------------------------------------------------------------------+ | wsrep_local_state_uuid | 93b28702-afa1-11ea-b0fe-13cd618aca8b | | wsrep_protocol_version | 10 | | wsrep_last_committed | 2 | | wsrep_replicated | 1 | | wsrep_replicated_bytes | 528 | | wsrep_repl_keys | 1 | | wsrep_repl_keys_bytes | 32 | | wsrep_repl_data_bytes | 425 | | wsrep_repl_other_bytes | 0 | | wsrep_received | 2 | | wsrep_received_bytes | 136 | | wsrep_local_commits | 0 | | wsrep_local_cert_failures | 0 | | wsrep_local_replays | 0 | | wsrep_local_send_queue | 0 | | wsrep_local_send_queue_max | 2 | | wsrep_local_send_queue_min | 0 | | wsrep_local_send_queue_avg | 0.333333 | | wsrep_local_recv_queue | 0 | | wsrep_local_recv_queue_max | 1 | | wsrep_local_recv_queue_min | 0 | | wsrep_local_recv_queue_avg | 0 | | wsrep_local_cached_downto | 1 | | wsrep_flow_control_paused_ns | 0 | | wsrep_flow_control_paused | 0 | | wsrep_flow_control_sent | 0 | | wsrep_flow_control_recv | 0 | | wsrep_cert_deps_distance | 1 | | wsrep_apply_oooe | 0 | | wsrep_apply_oool | 0 | | wsrep_apply_window | 1 | | wsrep_commit_oooe | 0 | | wsrep_commit_oool | 0 | | wsrep_commit_window | 1 | | wsrep_local_state | 4 | | wsrep_local_state_comment | Synced | | wsrep_cert_index_size | 1 | | wsrep_causal_reads | 0 | | wsrep_cert_interval | 0 | | wsrep_open_transactions | 0 | | wsrep_open_connections | 0 | | wsrep_incoming_addresses | AUTO | | wsrep_cluster_weight | 1 | | wsrep_desync_count | 0 | | wsrep_evs_delayed | | | wsrep_evs_evict_list | | | wsrep_evs_repl_latency | 0/0/0/0/0 | | wsrep_evs_state | OPERATIONAL | | wsrep_gcomm_uuid | 93b23072-afa1-11ea-a2a7-9f334cafb9f0 | | wsrep_applier_thread_count | 1 | | wsrep_cluster_capabilities | | | wsrep_cluster_conf_id | 1 | | wsrep_cluster_size | 1 | | wsrep_cluster_state_uuid | 93b28702-afa1-11ea-b0fe-13cd618aca8b | | wsrep_cluster_status | Primary | | wsrep_connected | ON | | wsrep_local_bf_aborts | 0 | | wsrep_local_index | 0 | | wsrep_provider_capabilities | :MULTI_MASTER:CERTIFICATION:PARALLEL_APPLYING:TRX_REPLAY:ISOLATION:PAUSE:CAUSAL_READS:INCREMENTAL_WRITESET:UNORDERED:PREORDERED:STREAMING:NBO: | | wsrep_provider_name | Galera | | wsrep_provider_vendor | Codership Oy | | wsrep_provider_version | 26.4.4(r4599) | | wsrep_ready | ON | | wsrep_rollbacker_thread_count | 1 | | wsrep_thread_count | 2 | +-------------------------------+------------------------------------------------------------------+ 65 rows in set (0.000 sec) MariaDB [(none)]> use mysql; Reading table information for completion of table and column names You can turn off this feature to get a quicker startup with -A Database changed MariaDB [mysql]> select * from wsrep_cluster; +--------------------------------------+---------+------------+------------------+--------------+ | cluster_uuid | view_id | view_seqno | protocol_version | capabilities | +--------------------------------------+---------+------------+------------------+--------------+ | 93b28702-afa1-11ea-b0fe-13cd618aca8b | 1 | 1 | 4 | 184703 | +--------------------------------------+---------+------------+------------------+--------------+ 1 row in set (0.000 sec) MariaDB [mysql]> select * from wsrep_cluster_members; +--------------------------------------+--------------------------------------+------------+-----------------------+ | node_uuid | cluster_uuid | node_name | node_incoming_address | +--------------------------------------+--------------------------------------+------------+-----------------------+ | d439b3e1-afa5-11ea-afdf-8e04ea7c7874 | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db1.galera | AUTO | +--------------------------------------+--------------------------------------+------------+-----------------------+ 1 row in set (0.000 sec) MariaDB [mysql]> select * from wsrep_streaming_log; Empty set (0.000 sec) ----------------------------------------------------- 두번째 노드부터 ]# systemctl start mariadb.service ]# mysql -uroot -p Enter password: Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 13 Server version: 10.4.13-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> use mysql; Reading table information for completion of table and column names You can turn off this feature to get a quicker startup with -A Database changed MariaDB [mysql]> select * from wsrep_cluster; +--------------------------------------+---------+------------+------------------+--------------+ | cluster_uuid | view_id | view_seqno | protocol_version | capabilities | +--------------------------------------+---------+------------+------------------+--------------+ | 93b28702-afa1-11ea-b0fe-13cd618aca8b | 2 | 4 | 4 | 184703 | +--------------------------------------+---------+------------+------------------+--------------+ 1 row in set (0.000 sec) MariaDB [mysql]> select * from wsrep_cluster_members; +--------------------------------------+--------------------------------------+------------+-----------------------+ | node_uuid | cluster_uuid | node_name | node_incoming_address | +--------------------------------------+--------------------------------------+------------+-----------------------+ | ab9dba35-afa6-11ea-9204-f6c024e4fb9d | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db2.galera | AUTO | | d439b3e1-afa5-11ea-afdf-8e04ea7c7874 | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db1.galera | AUTO | +--------------------------------------+--------------------------------------+------------+-----------------------+ 2 rows in set (0.000 sec) ----------------------------------------------------- 세번째 노드 ]# systemctl start mariadb.service ]# mysql -uroot -p Enter password: Welcome to the MariaDB monitor. Commands end with ; or \g. Your MariaDB connection id is 13 Server version: 10.4.13-MariaDB MariaDB Server Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. MariaDB [(none)]> use mysql; Reading table information for completion of table and column names You can turn off this feature to get a quicker startup with -A Database changed MariaDB [mysql]> select * from wsrep_cluster; +--------------------------------------+---------+------------+------------------+--------------+ | cluster_uuid | view_id | view_seqno | protocol_version | capabilities | +--------------------------------------+---------+------------+------------------+--------------+ | 93b28702-afa1-11ea-b0fe-13cd618aca8b | 3 | 5 | 4 | 184703 | +--------------------------------------+---------+------------+------------------+--------------+ 1 row in set (0.000 sec) MariaDB [mysql]> select * from wsrep_cluster_members; +--------------------------------------+--------------------------------------+------------+-----------------------+ | node_uuid | cluster_uuid | node_name | node_incoming_address | +--------------------------------------+--------------------------------------+------------+-----------------------+ | ab9dba35-afa6-11ea-9204-f6c024e4fb9d | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db2.galera | AUTO | | d439b3e1-afa5-11ea-afdf-8e04ea7c7874 | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db1.galera | AUTO | | f461131d-afa6-11ea-8c88-8644af0cf405 | 93b28702-afa1-11ea-b0fe-13cd618aca8b | db3.galera | AUTO | +--------------------------------------+--------------------------------------+------------+-----------------------+ 3 rows in set (0.000 sec) ----------------------------------------------------- ]# galera_new_cluster Job for mariadb.service failed because the control process exited with error code. See "systemctl status mariadb.service" and "journalctl -xe" for details. ]# tail -n 100 /var/log//messages (......) Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [Note] WSREP: Service thread queue flushed. Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [Note] WSREP: ####### Assign initial position for certification: c2318e3b-b128-11ea-af11-9f2a4a26b2a8:19, protocol version: -1 Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [Note] WSREP: Start replication Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [Note] WSREP: Connecting with bootstrap option: 1 Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [Note] WSREP: Setting GCS initial position to c2318e3b-b128-11ea-af11-9f2a4a26b2a8:19 Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [ERROR] WSREP: It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 . Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [ERROR] WSREP: wsrep::connect(gcomm://192.168.0.102,192.168.0.103,192.168.0.104) failed: 7 Jun 19 14:44:48 C8-DB-01 mysqld[1495]: 2020-06-19 14:44:48 0 [ERROR] Aborting Jun 19 14:44:48 C8-DB-01 systemd[1]: mariadb.service: Main process exited, code=exited, status=1/FAILURE Jun 19 14:44:48 C8-DB-01 systemd[1]: mariadb.service: Failed with result 'exit-code'. Jun 19 14:44:48 C8-DB-01 systemd[1]: Failed to start MariaDB 10.4.13 database server. ]# cat /var/lib/mysql/grastate.dat # GALERA saved state version: 2.1 uuid: c2318e3b-b128-11ea-af11-9f2a4a26b2a8 seqno: -1 safe_to_bootstrap: 0 safe_to_bootstrap: 0 -----> safe_to_bootstrap: 1 ]# galera_new_cluster ]# tail -n 100 /var/log//messages (......) Jun 19 14:58:23 C8-DB-01 mysqld[1740]: 2020-06-19 14:58:23 0 [Note] Added new Master_info '' to hash table Jun 19 14:58:23 C8-DB-01 mysqld[1740]: 2020-06-19 14:58:23 0 [Note] /usr/sbin/mysqld: ready for connections. Jun 19 14:58:23 C8-DB-01 mysqld[1740]: Version: '10.4.13-MariaDB' socket: '/tmp/galera.sock' port: 30306 MariaDB Server Jun 19 14:58:23 C8-DB-01 systemd[1]: Started MariaDB 10.4.13 database server. ㅇ LB 환경설정 ]# cat /etc/selinux/config SELINUX=disabled ]# echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf ]# sysctl -p 패키지 설치 ]# dnf install -y keepalived ipvsadm 설치됨: ipvsadm-1.31-1.el8.x86_64 keepalived-2.0.10-10.el8.x86_64 lm_sensors-libs-3.4.0-21.20180522git70f7e08.el8.x86_64 mariadb-connector-c-3.0.7-1.el8.x86_64 mariadb-connector-c-config-3.0.7-1.el8.noarch net-snmp-agent-libs-1:5.8-14.el8.x86_64 net-snmp-libs-1:5.8-14.el8.x86_64 perl-Carp-1.42-396.el8.noarch perl-Exporter-5.72-396.el8.noarch perl-libs-4:5.26.3-416.el8.x86_64 완료되었습니다! 서버 상태에 따른 스크립트 설정 스탠바이 서버가 backup 상태로 돌입 할 때 실행 ]# cat /etc/keepalived/backup #!/bin/sh echo "I'm LVS Backup" > /tmp/stat.txt 스탠바이 서버가 master 상태로 돌입 할 때 실행 ]# cat /etc/keepalived/master #!/bin/sh echo "I'm LVS Master" > /tmp/stat.txt ]# chmod 755 /etc/keepalived/backup ]# chmod 755 /etc/keepalived/master --------------------------------------- ㅇ DB 서버 상태 감지 ]# curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | sudo bash ]# dnf install MariaDB-client ]# cat /etc/keepalived/mariadb-check.sh #!/bin/bash mysql --host=$1 --port=30306 --user=lvs_test --password=lvs_test! lvs_test -Nse "SELECT * FROM tLVS ORDER BY wdate DESC limit 1" ]# chmod 755 /etc/keepalived/mariadb-check.sh ]# cd /etc/keepalived/ ]# mv keepalived.conf keepalived.conf.default ㅇ LVS 마스터 설정 ]# cat /etc/keepalived/keepalived.conf global_defs { notification_email { } router_id LVS_DB } vrrp_instance VI_2 { state MASTER interface enp0s3 garp_master_delay 10 virtual_router_id 40 priority 200 advert_int 1 authentication { auth_type PASS auth_pass 1306 } virtual_ipaddress { 192.168.0.101 } notify_backup /etc/keepalived/backup notify_master /etc/keepalived/master } virtual_server 192.168.0.101 30306 { delay_loop 3 lb_algo lc lb_kind DR protocol TCP #persistence_timeout 30 connect_timeout 5 real_server 192.168.0.102 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.102" misc_timeout 10 } } real_server 192.168.0.103 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.103" misc_timeout 10 } } real_server 192.168.0.104 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.104" misc_timeout 10 } } } --------------------------------------- ㅇ LVS 스탠바이 설정 ]# cat /etc/keepalived/keepalived.conf global_defs { notification_email { } router_id LVS_DB } vrrp_instance VI_2 { state BACKUP interface enp0s3 garp_master_delay 10 virtual_router_id 40 priority 100 advert_int 1 authentication { auth_type PASS auth_pass 1306 } virtual_ipaddress { 192.168.0.101 } notify_backup /etc/keepalived/backup notify_master /etc/keepalived/master } virtual_server 192.168.0.101 30306 { delay_loop 3 lb_algo lc lb_kind DR protocol TCP #persistence_timeout 30 connect_timeout 5 real_server 192.168.0.102 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.102" misc_timeout 10 } } real_server 192.168.0.103 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.103" misc_timeout 10 } } real_server 192.168.0.104 30306 { weight 10 MISC_CHECK { misc_path "/etc/keepalived/mariadb-check.sh 192.168.0.104" misc_timeout 10 } } } --------------------------------------- ㅇ Real Server (DB Server) 커널 파라미터 변경 기본값 ]# cat /proc/sys/net/ipv4/conf/enp0s3/arp_ignore 0 ]# cat /proc/sys/net/ipv4/conf/enp0s3/arp_announce 0 ]# vi /etc/sysctl.conf net.ipv4.conf.enp0s3.arp_ignore = 1 net.ipv4.conf.enp0s3.arp_announce = 2 ]# sysctl -p net.ipv4.ip_forward = 1 net.ipv4.conf.ens18.arp_ignore = 1 net.ipv4.conf.ens18.arp_announce = 2 net.ipv4.conf.ens19.arp_ignore = 1 net.ipv4.conf.ens19.arp_announce = 2 loop back IP 등록 - 각각의 real서버에 아래와 같이 vip를 loop back IP로 등록해 주거나, /etc/sysconfig/network-scripts/ifcfg-lo:0 파일을 생성해서 설정하고 network restart 한다. ifconfig lo:0 192.168.0.101 netmask 255.255.255.255 broadcast 192.168.0.101 up ]# cat /etc/sysconfig/network-scripts/ifcfg-lo:0 DEVICE=lo:0 IPADDR=192.168.0.101 NETMASK=255.255.255.255 NETWORK=192.168.0.0 BROADCAST=192.168.0.101 ONBOOT=yes NAME=loopback ]# service network restart ]# cat ifcfg-enp0s3:0 TYPE=Ethernet PROXY_METHOD=none BROWSER_ONLY=no BOOTPROTO=none IPADDR=192.168.0.101 PREFIX=24 DEFROUTE=yes IPV4_FAILURE_FATAL=no IPV6INIT=no NAME=enp0s3:0 DEVICE=enp0s3:0 ONBOOT=yes ㅇ 모니터링용 테이블 생성 CREATE TABLE `tLVS` ( `ano` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, `wdate` DATETIME NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), PRIMARY KEY (`ano`) USING BTREE ) COMMENT='tLVS' COLLATE='latin1_swedish_ci' ENGINE=InnoDB ; ㅇ LVS 방화벽 설정 firewalld 에서 VRRP 가 기본으로 허용되어 있지 않음. (멀티캐스트 224.0.0.18) VRRP 가 허용 되있지 않으면 keepalived 노드간 헬스 체크가 불가능해서 각각 master로 동작함. (Split Brain) ]# firewall-cmd --permanent --zone=public --add-port=30306/tcp success ]# firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept' --permanent ]# firewall-cmd --reload 또는, ]# firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0 --in-interface eth0 --destination 224.0.0.18 --protocol vrrp -j ACCEPT ]# firewall-cmd --direct --permanent --add-rule ipv4 filter OUTPUT 0 --out-interface eth0 --destination 224.0.0.18 --protocol vrrp -j ACCEPT ]# firewall-cmd --reload ]# firewall-cmd --info-zone=public public (active) target: default icmp-block-inversion: no interfaces: eth0 sources: services: ssh dhcpv6-client http https ports: protocols: masquerade: no forward-ports: source-ports: icmp-blocks: rich rules: rule protocol value="vrrp" accept ㅇ LVS 기본값 설정 : rc.local ]# cat /etc/rc.local ipvsadm --set 10 10 10 ]# chmod +x /etc/rc.d/rc.local ]# cat /usr/lib/systemd/system/rc-local.service (......) [Install] WantedBy=multi-user.target ]# systemctl daemon-reload ]# systemctl start rc-local.service ]# systemctl status rc-local.service ]# systemctl enable rc-local.service ]# watch -n 1 "ipvsadm -L" Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP LB-03:30306 lc -> 192.168.5.41:30306 Route 10 5 0 -> 192.168.5.42:30306 Route 10 5 0 -> 192.168.5.43:30306 Route 10 6 0 ActiveConn 값이 10초 뒤에 줄어들면 정상 재부팅해서도 같은지 확인. ㅇ LVS keepalived 데몬 실행 ]# service keepalived start ]# systemctl start keepalived ]# systemctl status keepalived ]# systemctl enable keepalived ]# systemctl reload keepalived ]# ipvsadm -L IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP LB-03:30306 rr -> 192.168.5.41:30306 Route 10 0 0 -> 192.168.5.42:30306 Route 10 0 0 -> 192.168.5.43:30306 Route 10 0 0 ifconfig 에는 가상 IP가 보이지 않음. ip addr 로 확인 가능. ]# ifconfig eth0: flags=4163 mtu 1500 inet 192.168.5.61 netmask 255.255.255.0 broadcast 192.168.5.255 inet6 fe80::18be:fdff:fed7:1a1c prefixlen 64 scopeid 0x20 ether 1a:be:fd:d7:1a:1c txqueuelen 1000 (Ethernet) RX packets 383117 bytes 23159404 (22.0 MiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 6180 bytes 526097 (513.7 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 lo: flags=73 mtu 65536 inet 127.0.0.1 netmask 255.0.0.0 inet6 ::1 prefixlen 128 scopeid 0x10 loop txqueuelen 1 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 ]# ip addr 1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 3e:e0:f9:05:62:f1 brd ff:ff:ff:ff:ff:ff inet 192.168.5.63/24 brd 192.168.5.255 scope global noprefixroute eth0 valid_lft forever preferred_lft forever inet 192.168.5.40/32 scope global eth0 <------ VIP valid_lft forever preferred_lft forever inet6 fe80::3ce0:f9ff:fe05:62f1/64 scope link valid_lft forever preferred_lft forever ]# ipvsadm -l IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP LB-03:30306 rr -> 192.168.5.41:30306 Route 10 3 0 -> 192.168.5.42:30306 Route 10 3 0 -> 192.168.5.43:30306 Route 10 4 0 ]# watch -n 1 "ipvsadm -l -n" ㅇ LVS Log 설정 ]# vi /etc/sysconfig/keepalived #KEEPALIVED_OPTIONS="-D" KEEPALIVED_OPTIONS="-D -d -S 0" ]# vi /etc/rsyslog.conf # Keepalived local0.* /var/log/keepalived.log ]# systemctl restart rsyslog ]# systemctl restart keepalived ]# cat /var/log/keepalived.log