本文目标:配置harbor一主二从（master 192.168.117.129,slave 192.168.117.130,192.168.117.131）,其中harbor-master外挂一主两从的数据库mysql、redis。

部署docker

安装包下载链接：链接: https://pan.baidu.com/s/1dqi0IQlILkfQpjmDqxTktw 提取码: p3h3
1）将docker1.12.6的rpm安装包上传到/home目录
2）本地安装docker

1
2
3
4
cd /home/docker1.12.6
yum localinstall *.rpm
systemctl start docker
systemctl enable docker  #设置开机自启动

如果 yum localinstall *.rpm显示缺少相关依赖，可重新配置一个本地yum源。也可使用rpm -ivh *.rpm –nodeps –force强制安装，但尽量不要使用，因为这样方式安装的docker可能有问题，导致装harbor的时候报错。配置yum源参考：https://blog.csdn.net/shida_csdn/article/details/78477202

部署harbor

（1）将harbor安装包上传到/home,解压
（2）根据实际情况修改harbor.cfg(master外挂mysql，从暂时不挂)

1
vim harbor.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
填写VIP或FQDN。请不要使用 localhost 或 127.0.0.1 类似的地址。
Hostname = 192.168.117.129

使用https协议请在`ssl_cert`和`ssl_cert_key`添加相应的证书。默认http协议。
ui_url_protocol = http

设置job service数量，建议50。请注意job会消耗网络、CPU和磁盘的资源
max_job_workers = 50

设置harbor管理员密码
harbor_admin_password = Harbor12345

关闭帐号注册功能。设置为off。
self_registration = off

设置外部数据库主机地址
db_host = 192.168.117.129

设置外部数据库root帐号密码
db_password = 123456

设置外部数据库端口
db_port = 3306

设置外部数据库管理员帐号
db_user = root

设置redis数据库地址(IP:port,权重,pwd)
redis_url = 192.168.117.129:6379,1,redis-2019

设置registry存储目录
#registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
registry_storage_provider_name = filesystem
#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
registry_storage_provider_config =

【说明】registry_storage_provider_name设置filesystem，必须创建/data/registry目录并将其所有者更改为10000:10000，因为harbor将以userID 10000和groupID 10000的形式运行。具体可参考官方文档。https://github.com/vmware/harbor/blob/master/docs/high_availability_installation_guide.md
（3）在外挂的数据库中建立registry数据库，并导入harbor schema

1
2
3
4
5
mysql -uroot -p
输入mysql登录密码
CREATE database registry;
use registry #选择要操作的数据库
执行下述sql语句
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
  create table access (
  access_id int NOT NULL AUTO_INCREMENT,
  access_code char(1),
  comment varchar (30),
  primary key (access_id)
);
insert into access (access_code, comment) values
('M', 'Management access for project'),
('R', 'Read access for project'),
('W', 'Write access for project'),
('D', 'Delete access for project'),
('S', 'Search access for project');

create table role (
  role_id int NOT NULL AUTO_INCREMENT,
  role_mask int DEFAULT 0 NOT NULL,
  role_code varchar(20),
  name varchar (20),
  primary key (role_id)
  );
insert into role (role_code, name) values
  ('MDRWS', 'projectAdmin'),
  ('RWS', 'developer'),
  ('RS', 'guest');

create table user (
  user_id int NOT NULL AUTO_INCREMENT,
  username varchar(255),
  email varchar(255),
  password varchar(40) NOT NULL,
  realname varchar (255) NOT NULL,
  comment varchar (30),
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  reset_uuid varchar(40) DEFAULT NULL,
  salt varchar(40) DEFAULT NULL,
  sysadmin_flag tinyint (1),
  creation_time timestamp NOT NULL default CURRENT_TIMESTAMP,
  update_time timestamp NOT NULL default CURRENT_TIMESTAMP,
  primary key (user_id),
  UNIQUE (username),
  UNIQUE (email)
 );
 insert into user (username, email, password, realname, comment, deleted, sysadmin_flag, creation_time, update_time) values ('admin', 'admin@example.com', '', 'system admin', 'admin user',0, 1, NOW(), NOW()),('anonymous','anonymous@example.com', '', 'anonymous user', 'anonymous user', 1, 0, NOW(), NOW());

create table project (
   project_id int NOT NULL AUTO_INCREMENT,
   owner_id int NOT NULL,
   name varchar (255) NOT NULL,
   creation_time timestamp NOT NULL default CURRENT_TIMESTAMP,
   update_time timestamp NOT NULL default CURRENT_TIMESTAMP,
   deleted tinyint (1) DEFAULT 0 NOT NULL,
   primary key (project_id),
   FOREIGN KEY (owner_id) REFERENCES user(user_id),
   UNIQUE (name)
  );
insert into project (owner_id, name, creation_time, update_time) values
 (1, 'library', NOW(), NOW());

create table project_member (
  project_id int NOT NULL,
  user_id int NOT NULL,
  role int NOT NULL,
  creation_time timestamp NOT NULL default CURRENT_TIMESTAMP,
  update_time timestamp NOT NULL default CURRENT_TIMESTAMP,
  PRIMARY KEY (project_id, user_id),
  FOREIGN KEY (role) REFERENCES role(role_id),
  FOREIGN KEY (project_id) REFERENCES project(project_id),
  FOREIGN KEY (user_id) REFERENCES user(user_id)
);
insert into project_member (project_id, user_id, role, creation_time, update_time) values(1, 1, 1, NOW(), NOW());

create table project_metadata (
  id int NOT NULL AUTO_INCREMENT,
  project_id int NOT NULL,
  name varchar(255) NOT NULL,
  value varchar(255),
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  deleted tinyint (1) DEFAULT 0 NOT NULL,
  PRIMARY KEY (id),
  CONSTRAINT unique_project_id_and_name UNIQUE (project_id,name),
  FOREIGN KEY (project_id) REFERENCES project(project_id)
);
insert into project_metadata (id, project_id, name, value, creation_time, update_time, deleted) values
(1, 1, 'public', 'true', NOW(), NOW(), 0);

create table access_log (
   log_id int NOT NULL AUTO_INCREMENT,
   username varchar (255) NOT NULL,
   project_id int NOT NULL,
   repo_name varchar (256),
   repo_tag varchar (128),
   GUID varchar(64),
   operation varchar(20) NOT NULL,
   op_time timestamp NOT NULL default CURRENT_TIMESTAMP,
   primary key (log_id),
   INDEX pid_optime (project_id, op_time)
);

create table repository (
  repository_id int NOT NULL AUTO_INCREMENT,
  name varchar(255) NOT NULL,
  project_id int NOT NULL,
  description text,
  pull_count int DEFAULT 0 NOT NULL,
  star_count int DEFAULT 0 NOT NULL,
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  primary key (repository_id),
  UNIQUE (name)
);

create table replication_policy (
   id int NOT NULL AUTO_INCREMENT,
   name varchar(256),
   project_id int NOT NULL,
   target_id int NOT NULL,
   enabled tinyint(1) NOT NULL DEFAULT 1,
   description text,
   deleted tinyint (1) DEFAULT 0 NOT NULL,
   cron_str varchar(256),
   filters varchar(1024),
   replicate_deletion tinyint (1) DEFAULT 0 NOT NULL,
   start_time timestamp NULL,
   creation_time timestamp default CURRENT_TIMESTAMP,
   update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
   PRIMARY KEY (id)
);

create table replication_target (
   id int NOT NULL AUTO_INCREMENT,
   name varchar(64),
   url varchar(64),
   username varchar(255),
   password varchar(128),
   target_type tinyint(1) NOT NULL DEFAULT 0,
   insecure tinyint(1) NOT NULL DEFAULT 0,
   creation_time timestamp default CURRENT_TIMESTAMP,
   update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
   PRIMARY KEY (id)
  );

 create table replication_job (
   id int NOT NULL AUTO_INCREMENT,
   status varchar(64) NOT NULL,
   policy_id int NOT NULL,
   repository varchar(256) NOT NULL,
   operation  varchar(64) NOT NULL,
   tags   varchar(16384),
   creation_time timestamp default CURRENT_TIMESTAMP,
   update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
   PRIMARY KEY (id),
   INDEX policy (policy_id),
   INDEX poid_uptime (policy_id, update_time)
  );

create table replication_immediate_trigger (
   id int NOT NULL AUTO_INCREMENT,
   policy_id int NOT NULL,
   namespace varchar(256) NOT NULL,
   on_push tinyint(1) NOT NULL DEFAULT 0,
   on_deletion tinyint(1) NOT NULL DEFAULT 0,
   creation_time timestamp default CURRENT_TIMESTAMP,
   update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
   PRIMARY KEY (id)
 );

create table img_scan_job (
  id int NOT NULL AUTO_INCREMENT,
  status varchar(64) NOT NULL,
  repository varchar(256) NOT NULL,
  tag   varchar(128) NOT NULL,
  digest varchar(128),
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  PRIMARY KEY (id)
);

create table img_scan_overview (
  id int NOT NULL AUTO_INCREMENT,
  image_digest varchar(128) NOT NULL,
  scan_job_id int NOT NULL,
  severity int NOT NULL default 0,
  components_overview varchar(2048),
  details_key varchar(128),
  creation_time timestamp default CURRENT_TIMESTAMP,
  update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  PRIMARY KEY(id),
  UNIQUE(image_digest)
);

create table clair_vuln_timestamp (
  id int NOT NULL AUTO_INCREMENT,
  namespace varchar(128) NOT NULL,
  last_update timestamp NOT NULL,
  PRIMARY KEY(id),
  UNIQUE(namespace)
);

create table properties (
   id int NOT NULL AUTO_INCREMENT,
   k varchar(64) NOT NULL,
   v varchar(128) NOT NULL,
   PRIMARY KEY(id),
   UNIQUE (k)
);

CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
insert into alembic_version values ('1.4.0');

启动harbor

1
2
3
4
cd /home/harbor
./prepare
./install --ha
docker ps

安装keepalived服务

Keepalived是一个免费开源的，用C编写的类似于layer3, 4 & 7交换机制软件，具备我们平时说的第3层、第4层和第7层交换机的功能。主要提供loadbalancing（负载均衡）和high-availability（高可用）功能，负载均衡实现需要依赖Linux的虚拟服务内核模块（ipvs），而高可用是通过VRRP协议实现多台机器之间的故障转移服务。
keepalived安装包下载链接: https://pan.baidu.com/s/13kW_Bz6RGSo4ewZ68BwCVA 提取码: 6ktn

i)设置每台机器的虚拟ip

1
ifconfig ens33:1 192.168.117.208 broadcast 192.168.117.129 netmask 255.255.255.0 up

image
将上述命令写在/etc/rc.local里进行开机自动设置
执行ifconfig查看是否生效
image
ii)修改keepalived的配置文件

1
2
yum localinstall *.rpm或rpm -ivh *.rpm --nodeps --force
vim /etc/keepalived/keepalived.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
! Configuration File for keepalived

global_defs {
   router_id harborlb          #更改内容
}
vrrp_sync_groups VG1 {
  group {
    VI_1
  }
}
vrrp_instance VI_1 {
    state MASTER       #谁被备份
    interface ens33    #网卡的名称
    virtual_router_id 51  #同一集群ID应该一致
    priority 100          #优先级，数字越高优先级越高，决定主备
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.117.208          #虚拟IP，与现有IP应不冲突
    }
    track_interface {
        ens33
   }
   notify_master "/etc/keepalived/redis_master.sh 192.168.117.129 6379"
   notify_backup "/etc/keepalived/redis_backup.sh 192.168.117.129 6379"
}
virtual_server 192.168.117.208 80 {
  delay_loop 15
  lb_algo rr
  lb_kind DR
  protocol TCP
  nat_mask 255.255.255.0
  persistence_timeout 10

  real_server 192.168.117.129 80 {
    weight 10
    MISC_CHECK {
        misc_path "/usr/local/bin/check.sh 192.168.117.129"
        misc_timeout 5
    }
  }

  real_server 192.168.117.130 80 {
    weight 10
    MISC_CHECK {
        misc_path "/usr/local/bin/check.sh 192.168.117.130"
        misc_timeout 5
    }
  }

  real_server 192.168.117.131 80 {
    weight 10
    MISC_CHECK {
        misc_path "/usr/local/bin/check.sh 192.168.117.131"
        misc_timeout 5
    }
  }
}

iii)设置harbor检查脚本

1
vim /usr/local/bin/check.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#!/bin/bash

set -e
#get protocol

#LOG=/var/log/keepalived_check.log
nodeip=$1
nodeaddress="http://${nodeip}"
http_code=`curl -s -o /dev/null -w "%{http_code}" ${nodeaddress}`

if [ $http_code == 200 ] ; then
  protocol="http"
elif [ $http_code == 301 ]
then
  protocol="https"
else
#  echo "`date +"%Y-%m-%d %H:%M:%S"` $1, CHECK_CODE=$http_code" >> $LOG
  exit 1
fi

systeminfo=`curl -k -o - -s ${protocol}://${nodeip}/api/systeminfo`

echo $systeminfo | grep "registry_url"
if [ $? != 0 ] ; then
  exit 1
fi
#TODO need to check Clair, but currently Clair status api is unreachable from LB.
# echo $systeminfo | grep "with_clair" | grep "true"
# if [ $? == 0 ] ; then
# clair is enabled
# do some clair check
# else
# clair is disabled
# fi

#check top api

http_code=`curl -k -s -o /dev/null -w "%{http_code}\n" ${protocol}://${nodeip}/api/repositories/top`
set +e
if [ $http_code == 200 ] ; then
  exit 0
else
  exit 1
fi

为检查脚本添加执行权限

1
chmod +x /usr/local/bin/check.sh

iv)为keepalived开启转发

1
2
3
4
5
6
[root@localhost ~]# vim /etc/sysctl.conf
 net.ipv4.ip_forward = 1
 net.ipv4.ip_nonlocal_bind = 1

[root@localhost ~]# sysctl -p
[root@localhost ~]# systemctl restart keepalived

image
（5）验证