Th技术开发白皮书

微服务环境安装说明

2021/6/18

# 1.K8S安装

kubeadm是官方社区推出的一个用于快速部署kubernetes集群的工具。

这个工具能通过两条指令完成一个kubernetes集群的部署

# 创建一个 Master 节点
kubeadm init

# 将一个 Node 节点加入到当前集群中
kubeadm join <Master节点的IP和端口 >

kubeadm是官方社区推出的一个用于快速部署kubernetes集群的工具。

这个工具能通过两条指令完成一个kubernetes集群的部署：

# 创建一个 Master 节点
$ kubeadm init

# 将一个 Node 节点加入到当前集群中
$ kubeadm join <Master节点的IP和端口 >

# 1.安装要求

在开始之前，部署Kubernetes集群机器需要满足以下几个条件：

  • 一台或多台机器，操作系统 CentOS7.x-86_x64
  • 硬件配置：2GB或更多RAM，2个CPU或更多CPU，硬盘30GB或更多
  • 可以访问外网，需要拉取镜像，如果服务器不能上网，需要提前下载镜像并导入节点
  • 禁止swap分区

# 2. 准备环境

角色 IP
master 192.168.0.10
node1 192.168.0.11
node2 192.168.0.12
# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

# 关闭selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config  # 永久
setenforce 0  # 临时

# 关闭swap
swapoff -a  # 临时
sed -ri 's/.*swap.*/#&/' /etc/fstab    # 永久

# 根据规划设置主机名
hostnamectl set-hostname <hostname>

# 在master添加hosts
cat >> /etc/hosts << EOF
192.168.0.10 k8smaster
192.168.0.11 k8snode1
192.168.0.12 k8snode2
EOF

# 将桥接的IPv4流量传递到iptables的链
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system  # 生效

# 时间同步
yum install ntpdate -y
ntpdate time.windows.com

# 3. 所有节点安装Docker/kubeadm/kubelet

Kubernetes默认CRI（容器运行时）为Docker，因此先安装Docker。

# 3.1 安装Docker

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum -y install docker-ce-18.06.1.ce-3.el7
systemctl enable docker && systemctl start docker
docker --version
Docker version 18.06.1-ce, build e68fc7a

mkdir -p /etc/docker
cat > /etc/docker/daemon.json << EOF
{
  "registry-mirrors": [
   "https://c7j78ln0.mirror.aliyuncs.com",
    "https://dockerhub.azk8s.cn",
    "https://docker.mirrors.ustc.edu.cn",
    "http://hub-mirror.c.163.com"
  ],
"insecure-registries": ["registry.cn-test.supdevops.com"],
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
    },
  "data-root": "/home/docker"
}
EOF

#重启docker
systemctl daemon-reload && systemctl restart docker

# 3.2 添加阿里云YUM软件源

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 3.3 安装kubeadm，kubelet和kubectl

由于版本更新频繁，这里指定版本号部署：

yum install -y kubelet-1.18.0 kubeadm-1.18.0 kubectl-1.18.0
#启动kubelet并设置开机启动
systemctl start kubelet && systemctl enable kubelet

# 4. 部署Kubernetes Master

说明：

在192.168.0.10（Master）执行，由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓库地址。

--apiserver-advertise-address 是你master节点的IP

--kubernetes-version 为安装的K8S版本

kubeadm init \
--apiserver-advertise-address=192.168.0.10 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.18.0 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--v=6
#重新初始化
kubeadm reset

使用kubectl工具

#配置kubelet环境变量
cat <<EOF >> /etc/profile
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
#让环境变量生效
. /etc/profile

# 5. 加入Kubernetes Node

在192.168.1.12/13（Node）执行。

向集群添加新节点，执行在kubeadm init输出的kubeadm join命令：

kubeadm join 192.168.0.10:6443 --token esce21.q6hetwm8si29qxwn \
--discovery-token-ca-cert-hash sha256:00603a05805807501d7181c3d60b478788408cfe6cedefedb1f97569708be9c5

默认token有效期为24小时，当过期之后，该token就不可用了。这时就需要重新创建token，操作如下：

kubeadm token create --print-join-command

# 6. 部署CNI网络插件

#下载插件yaml文件
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

默认镜像地址无法访问，sed命令修改为docker hub镜像仓库。

#部署网络插件
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

#查看是否安装成功显示running即安装成功
kubectl get pods -n kube-system

image-20210714141051932

# 2.MySQL安装

# 1.下载对应系统的yum源进行安装

[root@MySQL-master ~]# yum localinstall mysql80-community-release-el7-1.noarch.rpm -y
[root@MySQL-master ~]# yum repolist all | grep mysql

img

[root@MySQL-master ~]# yum -y install yum-utils
#禁止安装80版本
[root@MySQL-master ~]# yum-config-manager --disable mysql80-community 
#开启安装56版本
[root@MySQL-master ~]# yum-config-manager --enable mysql56-community
[root@MySQL-master ~]# yum install mysql-community-server -y

# 2.修改配置文件

[root@MySQL-master ~]# cp /etc/my.cnf /etc/my.cnf.bak
[root@MySQL-master ~]# mkdir /home/app/mysql
[root@MySQL-master ~]# vim /etc/my.cnf

配置文件

[mysqld]
#数据存储目录
datadir=/home/app/mysql
socket=/var/lib/mysql/mysql.sock

innodb_force_recovery=0
symbolic-links=0
character_set_server=utf8
#开启binlog加这个防止还原时函数报错
log_bin_trust_function_creators = 1
#其中 0：区分大小写，1：不区分大小写
lower_case_table_names=1
#开启binlog日志
server-id=1
log-bin=mysql-bin
log-bin-index=master-bin.index
#自动删除7天前的binlog日志
expire_logs_days=7
#开启慢日志
slow_query_log =1
slow_query_log_file=/home/app/mysql/slow.log

[mysqld]
port= 3306
default-storage-engine=INNODB
skip-external-locking
key_buffer_size = 512M
max_allowed_packet = 400M
table_open_cache = 512
sort_buffer_size = 8M
read_buffer_size = 8M
read_rnd_buffer_size = 6M
myisam_sort_buffer_size = 128M
join_buffer_size = 6M
thread_cache_size = 8
query_cache_type= 1
query_cache_size= 32M
thread_concurrency = 8
lower_case_table_names=1
max_connections=1000
binlog_format=mixed
innodb_buffer_pool_size = 64M
innodb_additional_mem_pool_size = 2M
innodb_log_file_size = 5M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 1
innodb_thread_concurrency=8
innodb_lock_wait_timeout=50
autocommit=on

[mysqldump]
quick
max_allowed_packet = 16M
[mysql]
no-auto-rehash
skip-secure-auth
[myisamchk]
key_buffer_size = 256M
sort_buffer_size = 256M
read_buffer = 4M
write_buffer = 4M
[mysqlhotcopy]
interactive-timeout
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

# 3.启动MySQL

#启动
[root@MySQL-master ~]# systemctl start mysqld 
#开机自启
[root@MySQL-master ~]# systemctl enable mysqld

# 4.给用户赋权

[root@MySQL-master ~]# mysql -uroot
grant all privileges  on *.* to root@'%' identified by "w3d56yu8r" WITH GRANT OPTION;
grant all privileges  on *.* to root@'127.0.0.1' identified by "w3d56yu8r" WITH GRANT OPTION; 
grant all privileges  on *.* to root@'localhost' identified by "w3d56yu8r" WITH GRANT OPTION;
flush privileges;

# 3.Redis安装

# 1.安装

#yum安装Redis
yum install -y redis
#创建备份目录
mkdir /home/app/redis/

# 2.配置文件修改

#修改配置文件
vim /etc/redis.conf
requirepass itonghui  #密码
appendonly yes #打开备份
dir /home/app/redis   #在home/app下创建备份
#bind 127.0.0.1   #注释一下允许所有IP访问
#DB数量
databases 256

# 3.启动

#启动Redis并设置开机启动
systemctl start redis && systemctl enable redis 
#查看日志
tail -f /var/log/redis/redis.log

# 4.Harbor安装

# 1.安装资源要求

安装参照 (opens new window)

安装需求

资源 要求
CPU 2 CPU
Mem 4GB
Disk 40GB
Python >=2.7
Docker engine >=1.10
Docker Compose >=1.6.0

# 2.安装Docker、Docker-Compose

# 2.1安装docker

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum -y install docker-ce-18.06.1.ce-3.el7
systemctl enable docker && systemctl start docker
docker --version
Docker version 18.06.1-ce, build e68fc7a

mkdir -p /etc/docker
cat > /etc/docker/daemon.json << EOF
{
  "registry-mirrors": [
   "https://c7j78ln0.mirror.aliyuncs.com",
    "https://dockerhub.azk8s.cn",
    "https://docker.mirrors.ustc.edu.cn",
    "http://hub-mirror.c.163.com"
  ],
"insecure-registries": ["registry.cn-test.supdevops.com"],
  "max-concurrent-downloads": 10,
  "max-concurrent-uploads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
    },
  "data-root": "/home/docker"
}
EOF

#重启docker
systemctl daemon-reload && systemctl restart docker

# 2.2升级Python

#下载安装包
v=3.7.0;wget https://npm.taobao.org/mirrors/python/$v/Python-$v.tar.xz
#解压安装
tar xvf Python-3.7.0.tar.xz
cd Python-3.7.0
./configure --prefix=/usr/local/python3
make && make install
#备份原来的版本
mv /usr/bin/python /usr/bin/python2.7.bak
mv /usr/bin/pip /usr/bin/pip.bak
#新版本做个软链接
ln -s /usr/local/python3/bin/python3.7 /usr/bin/python
ln -s /usr/local/python3/bin/pip3 /usr/bin/pip

注意：

这种方式升级Python会导致yum安装不能使用，如果不能用使用下面方法修改yum

#修改配置文件
vim /usr/bin/yum
#!/usr/bin/python
import sys
try:
    import yum

将上面的语句改为：

如果是2.7就写2.7根据原有的Python版本来具体判断

#!/usr/bin/python2.7
import sys
try:
    import yum

vim /usr/libexec/urlgrabber-ext-down
将第一个行#!/usr/bin/python” 改为 “#!/usr/bin/python2.7”即可。

# 2.3安装Docker-Compose

#安装pip
CentOS:
yum -y install epel-release
yum -y install python-pip
Ubuntu:
apt-get install python-pip -y
#修改pip源
mkdir ~/.pip
cat > ~/.pip/pip.conf << EOF
[global]
index-url = https://mirrors.aliyun.com/pypi/simple/
[install]
EOF
#升级pip
python -m pip install --upgrade pip

#安装docker-compose
pip install docker-compose
docker-compose -version
#如果没有docker-compose命令，做下软链接
ln -s /usr/local/python3/bin/docker-compose /usr/local/bin/

# 3.离线安装harbor

下载地址 (opens new window)

tar xvf harbor-online-installer-v1.6.3.tgz
cd harbor
#修改配置文件harbor.yml

#修改Hostname,http.port,注释掉https，更改harbor_admin_password

# Configuration file of Harbor

# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 192.168.1.111

# http related config
http:
  # port for http, default is 80. If https enabled, this port will redirect to https port
  port: 8089

# https related config
#https:
#  https port for harbor, default is 443
#  port: 443
#  # The path of cert and key files for nginx
#  certificate: /your/certificate/path
#  private_key: /your/private/key/path

# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
# external_url: https://reg.mydomain.com:8433

# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: xxxxxxx

# 4.在harbor目录，执行./install.sh

#2.2以上加上这个参数开启扫描器
./install.sh --with-trivy --with-chartmuseum

img

# 5.登陆

用户名：admin
密码：配置文件中填写的密码

# 6.重启

cd /data/harbor
docker-compose stop
docker-compose up -d

# 5.Nacos注册中心安装

官方网文档 (opens new window)

# 1.单机版安装

#解压安装包
tar zxvf nacos-server-1.4.2.tar.gz
cd /home/www/nacos/bin
#启动
sh startup.sh -m standalone
#查看日志
tail -f /home/www/nacos/logs/start.out
#后面得加/nacos进行访问，默认用户名密码都是nacos
http://192.168.0.196:10002/nacos

# 2.集群版本安装

img

解压安装包

tar zxvf nacos-server-1.4.2.tar.gz

创建数据库，nacos的conf目录下的nacos-mysql.sql在新建的数据库中执行

img

进入nacos的conf目录，编辑application.properties文件，增加数据库配置

spring.datasource.platform=mysql
db.num=1
### Connect URL of DB:
db.url.0=jdbc:mysql://192.168.0.199:10000/nacos_200?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC
db.user.0=root
db.password.0=w3d56yu8r

最后一步需要配置一份集群节点信息，配置文件在conf目录下的cluster.conf.example文件，我们进行重命名成cluster.conf。

然后编辑cluster.conf文件，增加3个节点的信息，格式为IP:PORT，三个目录都一致即可。

192.168.0.196:20003
192.168.0.200:20000
192.168.0.200:20001

启动的话直接到bin目录下，执行./startup.sh就可以了，默认就是集群模式，不需要加任何参数。

Nginx做负载均衡，负载均衡配置这里不做详解，可以参照Nginx负载均衡配置

upstream nacos {
  server 192.168.0.196:20003;
  server 192.168.0.200:20000;
  server 192.168.0.200:20001;
}

# 3.Nacos鉴权

开启鉴权之前，application.properties中的配置信息为：

### If turn on auth system:
nacos.core.auth.enabled=false

开启鉴权之后，application.properties中的配置信息为：

### If turn on auth system:
nacos.core.auth.enabled=true

说明:

在安装就Nacos之前需要配置jdk环境变量，建议JDK8，环境变量配置这里不做详解，具体参照环境变量配置。

# 6.ActiveMQ安装

#解压安装包
tar zxvf apache-activemq-5.14.5-bin.tar.gz
#进入目录
cd /home/app/pache-activemq-5.14.5/bin/linux-x86-x64
#启动调试模式
./activemq console
#调试模式不报错用和这个命令启动
./activemq start

说明:

在安装就ActiveMQ之前需要配置jdk环境变量，建议JDK8，环境变量配置这里不做详解，具体参照环境变量配置。

# 7.JDK环境变量配置

#解压
tar zxvf jdk-8u181-linux-x64.tar.gz -C /home/app

#配置环境变量
vim /etc/profile
export JAVA_HOME=/home/app/jdk1.8.0_181
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

#使环境变量生效
. /etc/profile

#验证环境变量是否生效
java -version

# 8.Minio对象存储安装

# 1.下载安装

点我下载安装包 (opens new window)

#设置AKSK
vim /etc/profile
export MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
export MINIO_SECRET_KEY=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY
. /etc/profile

# 2.启动

#后台启动
nohup ./minio server --address 0.0.0.0:10005 /home/itonghui > /dev/null 2>&1 &

说明：

--address 是服务的端口和IP

/home/itonghui 是桶存放的本地磁盘路径

# 9.Jenkins安装

# 1.下载jenkins

下载地址 (opens new window)

# 2.安装jenkins

#配置环境变量
vim /etc/profile
export JAVA_HOME=/home/app/jdk1.8.0_181
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH

#使环境变量生效
. /etc/profile

#验证环境变量是否生效
java -version

----安装jenkins----
mv jenkins.war /home/app/apache-tomcat-8.5.59/webapps/ROOT

# 3.修改jenkins工作目录

#设置jenkins的工作目录
vim /home/app/apache-tomcat-8.5.59/bin/catalina.sh
export JENKINS_HOME="/home/data/jenkins"
vim /etc/profile
export JENKINS_HOME=/home/data/jenkins
#让环境变量生效
. /etc/profile

# 4.插件安装

#设置下载插件地址
https://mirrors.tuna.tsinghua.edu.cn/jenkins/updates/update-center.json
#中文插件
localization-zh-cn

# 10.ELK安装

下载地址 (opens new window)

# 1.安装Elasticsearch

#安装
yum -y install elasticsearch-7.8.0-x86_64.rpm
#修改配置文件
sed -i '17s/#cluster.name: my-application/cluster.name: elk/' /etc/elasticsearch/elasticsearch.yml
sed -i '23s/#node.name: node-1/node.name: node-1/' /etc/elasticsearch/elasticsearch.yml
sed -i '55s/#network.host: 192.168.0.1/network.host: 0.0.0.0/' /etc/elasticsearch/elasticsearch.yml
#单节点需要在配置文件加上这句话
cluster.initial_master_nodes: ["node-1"]

#若修改数据路径，需给路径es用户权限
chown -R elasticsearch:elasticsearch /home/ELK/elasticsearch/
#配置环境变量
vim /usr/share/elasticsearch/bin/elasticsearch
# 添加以下代码
export JAVA_HOME=/home/app/jdk1.8.0_181/
export PATH=$JAVA_HOME/bin:$PATH

if [ -x "$JAVA_HOME/bin/java" ]; then
        JAVA="/home/app/jdk1.8.0_181/bin/java"
else
        JAVA=`which java`
fi
#或者
ln -s /home/app/jdk1.8.0_181/bin/java /usr/bin/java
#启动
systemctl daemon-reload && systemctl enable elasticsearch && systemctl start elasticsearch

ES配置文件

#
# Use a descriptive name for your cluster:
#
cluster.name: elk
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
node.name: node-1
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#收集数据存放目录
path.data: /home/ELK/elasticsearch/data
#
# Path to log files:
#
path.logs: /var/log/elasticsearch
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#cluster.initial_master_nodes: ["node-1"]
network.host: 0.0.0.0
#
# ---------------------------------- xpack -----------------------------------
#xpack.monitoring.history.duration: 1d
#关闭xpack监控
xpack.monitoring.enabled: false

# 2.安装Kibana

yum -y install kibana-7.8.0-x86_64.rpm
sed -i '7s/#server.host: "localhost"/server.host: "0.0.0.0"/' /etc/kibana/kibana.yml
sed -i '28s/#elasticsearch.hosts: .*/elasticsearch.hosts: ["http:\/\/127.0.0.1:9200"]/' /etc/kibana/kibana.yml

#清空防火墙规则
iptables -F 
service  iptables save
systemctl enable kibana
systemctl start kibana
systemctl restart kibana

#汉化
vim /etc/kibana/kibana.yml
i18n.locale: "zh-CN"
systemctl restart kibana

#访问验证(默认5601端口)

ip:5601

# 3.安装filebeat

tar zxvf filebeat-6.7.0-linux-x86_64.tar.gz
#前台启动
./filebeat -e -c filebeat.yml
#后台启动
nohup ./filebeat -e -c filebeat.yml > /dev/null 2>&1 &
#杀死重启
ps -axu | grep filebeat | grep -v grep | awk '{print 2}' | xargs kill -9

微服务部署更新说明