Commit fadc628d authored by root's avatar root

update

parent e1e04d48
name: harbor
version: dev
appVersion: dev
version: 1.0.0
appVersion: 1.7.0
description: An open source trusted cloud native registry that stores, signs, and scans content
keywords:
- docker
......
......@@ -35,6 +35,11 @@ docker pull goharbor/harbor-core:v1.7.5 && \
docker tag goharbor/harbor-core:v1.7.5 registry-vpc.cn-qingdao.aliyuncs.com/wod/harbor-core:v1.7.5 && \
docker push registry-vpc.cn-qingdao.aliyuncs.com/wod/harbor-core:v1.7.5
# goharbor/harbor-adminserver
docker pull goharbor/harbor-adminserver:v1.7.5 && \
docker tag goharbor/harbor-adminserver:v1.7.5 registry-vpc.cn-qingdao.aliyuncs.com/wod/harbor-adminserver:v1.7.5 && \
docker push registry-vpc.cn-qingdao.aliyuncs.com/wod/harbor-adminserver:v1.7.5
# goharbor/harbor-jobservice
docker pull goharbor/harbor-jobservice:v1.7.5 && \
docker tag goharbor/harbor-jobservice:v1.7.5 registry-vpc.cn-qingdao.aliyuncs.com/wod/harbor-jobservice:v1.7.5 && \
......
......@@ -53,7 +53,7 @@ Configure the followings items in `values.yaml`, you can also set them as parame
If you have no PVCs that can be shared across nodes, you can use external object storage to store images and charts and store the job logs in database. Set the `persistence.imageChartStorage.type` to the value you want to use and fill the corresponding section and set `jobservice.jobLogger` to `database`.
- **Replica**
Set `portal.replicas`, `core.replicas`, `jobservice.replicas`, `registry.replicas`, `chartmuseum.replicas`, `clair.replicas`, `notary.server.replicas` and `notary.signer.replicas` to `n`(`n`>=2).
Set `portal.replicas`, `adminserver.replicas`, `core.replicas`, `jobservice.replicas`, `registry.replicas`, `chartmuseum.replicas`, `clair.replicas`, `notary.server.replicas` and `notary.signer.replicas` to `n`(`n`>=2).
### Installation
Install the Harbor helm chart with a release name `my-release`:
......
This diff is collapsed.
......@@ -36,6 +36,22 @@ release: {{ .Release.Name }}
{{- end -}}
{{- end -}}
{{- define "harbor.autoGenCertForIngress" -}}
{{- if and (eq (include "harbor.autoGenCert" .) "true") (eq .Values.expose.type "ingress") -}}
{{- printf "true" -}}
{{- else -}}
{{- printf "false" -}}
{{- end -}}
{{- end -}}
{{- define "harbor.autoGenCertForNginx" -}}
{{- if and (eq (include "harbor.autoGenCert" .) "true") (ne .Values.expose.type "ingress") -}}
{{- printf "true" -}}
{{- else -}}
{{- printf "false" -}}
{{- end -}}
{{- end -}}
{{- define "harbor.database.host" -}}
{{- if eq .Values.database.type "internal" -}}
{{- template "harbor.database" . }}
......@@ -216,6 +232,10 @@ host:port,pool_size,password
{{- printf "%s-redis" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.adminserver" -}}
{{- printf "%s-adminserver" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.jobservice" -}}
{{- printf "%s-jobservice" (include "harbor.fullname" .) -}}
{{- end -}}
......@@ -248,32 +268,6 @@ host:port,pool_size,password
{{- printf "%s-nginx" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.ingress.core" -}}
{{- printf "%s-ingress-core" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.ingress.notary" -}}
{{- printf "%s-ingress-notary" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.certificate" -}}
{{- printf "%s-certificate" (include "harbor.fullname" .) -}}
{{- end -}}
{{- define "harbor.certificate-secret" -}}
{{- $tls := .Values.expose.tls -}}
{{- if $tls.secretName }}
{{- printf "%s" $tls.secretName -}}
{{- else }}
{{- printf "%s" (include "harbor.certificate" .) -}}
{{- end }}
{{- end -}}
{{- define "harbor.common-name" -}}
{{- $expose := .Values.expose }}
{{- if and (eq $expose.type "ingress") $expose.ingress.host }}
{{- printf "%s" $expose.ingress.host -}}
{{- else }}
{{- printf "%s" $expose.tls.commonName -}}
{{- end }}
{{- define "harbor.ingress" -}}
{{- printf "%s-ingress" (include "harbor.fullname" .) -}}
{{- end -}}
\ No newline at end of file
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ template "harbor.adminserver" . }}"
labels:
{{ include "harbor.labels" . | indent 4 }}
app: adminserver
data:
DATABASE_TYPE: "postgresql"
POSTGRESQL_HOST: "{{ template "harbor.database.host" . }}"
POSTGRESQL_PORT: "{{ template "harbor.database.port" . }}"
POSTGRESQL_USERNAME: "{{ template "harbor.database.username" . }}"
POSTGRESQL_DATABASE: "{{ template "harbor.database.coreDatabase" . }}"
POSTGRESQL_SSLMODE: "{{ template "harbor.database.sslmode" . }}"
EXT_ENDPOINT: "{{ .Values.externalURL }}"
CORE_URL: "http://{{ template "harbor.core" . }}"
JOBSERVICE_URL: "http://{{ template "harbor.fullname" . }}-jobservice"
REGISTRY_URL: "http://{{ template "harbor.registry" . }}:5000"
TOKEN_SERVICE_URL: "http://{{ template "harbor.core" . }}/service/token"
WITH_NOTARY: "{{ .Values.notary.enabled }}"
NOTARY_URL: "http://{{ template "harbor.notary-server" . }}:4443"
LOG_LEVEL: "{{ .Values.logLevel }}"
IMAGE_STORE_PATH: "/" # This is a temporary hack.
CFG_EXPIRATION: "5"
ADMIRAL_URL: "NA"
RESET: "false"
WITH_CLAIR: "{{ .Values.clair.enabled }}"
CLAIR_DB_HOST: "{{ template "harbor.database.host" . }}"
CLAIR_DB_PORT: "{{ template "harbor.database.port" . }}"
CLAIR_DB_USERNAME: "{{ template "harbor.database.username" . }}"
CLAIR_DB: "{{ template "harbor.database.clairDatabase" . }}"
CLAIR_URL: "http://{{ template "harbor.fullname" . }}-clair:6060"
CLAIR_DB_SSLMODE: "{{ template "harbor.database.sslmode" . }}"
UAA_ENDPOINT: ""
UAA_CLIENTID: ""
UAA_CLIENTSECRET: ""
UAA_VERIFY_CERT: "True"
REGISTRY_STORAGE_PROVIDER_NAME: "{{ .Values.persistence.imageChartStorage.type }}"
WITH_CHARTMUSEUM: "{{ .Values.chartmuseum.enabled }}"
CHART_REPOSITORY_URL: "http://{{ template "harbor.chartmuseum" . }}"
# The following properties have no default value in Harbor core if not
# set via env. They should be removed after the refactoring:
# AUTH_MODE, SELF_REGISTRATION, PROJECT_CREATION_RESTRICTION
AUTH_MODE: "db_auth"
SELF_REGISTRATION: "on"
PROJECT_CREATION_RESTRICTION: "everyone"
TOKEN_EXPIRATION: "30"
EMAIL_HOST: "smtp.mydomain.com"
EMAIL_FROM: "admin <sample_admin@mydomain.com>"
EMAIL_PORT: "25"
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ template "harbor.adminserver" . }}"
labels:
{{ include "harbor.labels" . | indent 4 }}
app: adminserver
spec:
replicas: {{ .Values.adminserver.replicas }}
selector:
matchLabels:
{{ include "harbor.matchLabels" . | indent 6 }}
app: adminserver
template:
metadata:
labels:
{{ include "harbor.labels" . | indent 8 }}
app: adminserver
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/adminserver/adminserver-cm.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/adminserver/adminserver-secrets.yaml") . | sha256sum }}
checksum/secret-core: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
checksum/secret-jobservice: {{ include (print $.Template.BasePath "/jobservice/jobservice-secrets.yaml") . | sha256sum }}
{{- if .Values.adminserver.podAnnotations }}
{{ toYaml .Values.adminserver.podAnnotations | indent 8 }}
{{- end }}
spec:
containers:
- name: adminserver
image: "{{ .Values.adminserver.image.repository }}:{{ .Values.adminserver.image.tag }}"
imagePullPolicy: "{{ .Values.imagePullPolicy }}"
livenessProbe:
httpGet:
path: /api/ping
port: 8080
initialDelaySeconds: 1
periodSeconds: 10
readinessProbe:
httpGet:
path: /api/ping
port: 8080
initialDelaySeconds: 1
periodSeconds: 10
resources:
{{ toYaml .Values.adminserver.resources | indent 10 }}
envFrom:
- configMapRef:
name: "{{ template "harbor.adminserver" . }}"
- secretRef:
name: "{{ template "harbor.adminserver" . }}"
env:
- name: PORT
value: "8080"
- name: JSON_CFG_STORE_PATH
value: /etc/adminserver/config/config.json
- name: KEY_PATH
value: /etc/adminserver/key
- name: CORE_SECRET
valueFrom:
secretKeyRef:
name: "{{ template "harbor.core" . }}"
key: secret
- name: JOBSERVICE_SECRET
valueFrom:
secretKeyRef:
name: "{{ template "harbor.jobservice" . }}"
key: secret
ports:
- containerPort: 8080
volumeMounts:
- name: adminserver-key
mountPath: /etc/adminserver/key
subPath: key
- name: etc-localtime
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
path: /etc/localtime
- name: adminserver-key
secret:
secretName: "{{ template "harbor.adminserver" . }}"
items:
- key: secretKey
path: key
{{- with .Values.adminserver.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.adminserver.affinity }}
affinity:
{{ toYaml . | indent 8 }}
{{- end }}
{{- with .Values.adminserver.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
\ No newline at end of file
apiVersion: v1
kind: Secret
metadata:
name: "{{ template "harbor.adminserver" . }}"
labels:
{{ include "harbor.labels" . | indent 4 }}
app: adminserver
type: Opaque
data:
secretKey: {{ .Values.secretKey | b64enc | quote }}
HARBOR_ADMIN_PASSWORD: {{ .Values.harborAdminPassword | b64enc | quote }}
POSTGRESQL_PASSWORD: {{ template "harbor.database.encryptedPassword" . }}
{{ if .Values.clair.enabled }}
CLAIR_DB_PASSWORD: {{ template "harbor.database.encryptedPassword" . }}
{{ end }}
apiVersion: v1
kind: Service
metadata:
name: "{{ template "harbor.adminserver" . }}"
spec:
ports:
- port: 80
targetPort: 8080
selector:
{{ include "harbor.matchLabels" . | indent 4 }}
app: adminserver
\ No newline at end of file
......@@ -22,7 +22,7 @@ data:
DISABLE_API: "false"
DISABLE_STATEFILES: "false"
ALLOW_OVERWRITE: "true"
#CHART_URL: {{ .Values.externalURL }}/chartrepo
CHART_URL:
AUTH_ANONYMOUS_GET: "false"
TLS_CERT:
TLS_KEY:
......@@ -45,7 +45,6 @@ data:
{{- else if eq $storageType "gcs" }}
STORAGE: "google"
STORAGE_GOOGLE_BUCKET: {{ $storage.gcs.bucket }}
GOOGLE_APPLICATION_CREDENTIALS: /etc/chartmuseum/gcs-key.json
{{- if $storage.gcs.rootdirectory }}
STORAGE_GOOGLE_PREFIX: {{ $storage.gcs.rootdirectory }}
{{- end }}
......
......@@ -20,7 +20,6 @@ spec:
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/chartmuseum/chartmuseum-cm.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/chartmuseum/chartmuseum-secret.yaml") . | sha256sum }}
checksum/secret-core: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
{{- if .Values.chartmuseum.podAnnotations }}
{{ toYaml .Values.chartmuseum.podAnnotations | indent 8 }}
{{- end }}
......@@ -41,10 +40,8 @@ spec:
port: 9999
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.chartmuseum.resources }}
resources:
{{ toYaml .Values.chartmuseum.resources | indent 10 }}
{{- end }}
envFrom:
- configMapRef:
name: "{{ template "harbor.chartmuseum" . }}"
......@@ -62,7 +59,7 @@ spec:
- name: chartmuseum-data
mountPath: /chart_storage
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
......
......@@ -51,10 +51,8 @@ spec:
{{- end }}
- name: NO_PROXY
value: "{{ template "harbor.registry" . }},{{ template "harbor.core" . }}"
{{- if .Values.clair.resources }}
resources:
{{ toYaml .Values.clair.resources | indent 10 }}
{{- end }}
ports:
- containerPort: 6060
volumeMounts:
......
......@@ -7,10 +7,7 @@ metadata:
{{ include "harbor.labels" . | indent 4 }}
spec:
ports:
- name: clair
port: 6060
- name: health
port: 6061
- port: 6060
selector:
{{ include "harbor.matchLabels" . | indent 4 }}
app: clair
......
......@@ -7,41 +7,8 @@ metadata:
data:
app.conf: |+
appname = Harbor
runmode = dev
runmode = prod
enablegzip = true
[dev]
[prod]
httpport = 8080
DATABASE_TYPE: "postgresql"
POSTGRESQL_HOST: "{{ template "harbor.database.host" . }}"
POSTGRESQL_PORT: "{{ template "harbor.database.port" . }}"
POSTGRESQL_USERNAME: "{{ template "harbor.database.username" . }}"
POSTGRESQL_DATABASE: "{{ template "harbor.database.coreDatabase" . }}"
POSTGRESQL_SSLMODE: "{{ template "harbor.database.sslmode" . }}"
EXT_ENDPOINT: "{{ .Values.externalURL }}"
CORE_URL: "http://{{ template "harbor.core" . }}"
JOBSERVICE_URL: "http://{{ template "harbor.fullname" . }}-jobservice"
REGISTRY_URL: "http://{{ template "harbor.registry" . }}:5000"
TOKEN_SERVICE_URL: "http://{{ template "harbor.core" . }}/service/token"
WITH_NOTARY: "{{ .Values.notary.enabled }}"
NOTARY_URL: "http://{{ template "harbor.notary-server" . }}:4443"
CFG_EXPIRATION: "5"
WITH_CLAIR: "{{ .Values.clair.enabled }}"
CLAIR_DB_HOST: "{{ template "harbor.database.host" . }}"
CLAIR_DB_PORT: "{{ template "harbor.database.port" . }}"
CLAIR_DB_USERNAME: "{{ template "harbor.database.username" . }}"
CLAIR_DB: "{{ template "harbor.database.clairDatabase" . }}"
CLAIR_DB_SSLMODE: "{{ template "harbor.database.sslmode" . }}"
CLAIR_URL: "http://{{ template "harbor.fullname" . }}-clair:6060"
REGISTRY_STORAGE_PROVIDER_NAME: "{{ .Values.persistence.imageChartStorage.type }}"
WITH_CHARTMUSEUM: "{{ .Values.chartmuseum.enabled }}"
CHART_REPOSITORY_URL: "http://{{ template "harbor.chartmuseum" . }}"
LOG_LEVEL: "{{ .Values.logLevel }}"
CONFIG_PATH: "/etc/core/app.conf"
SYNC_REGISTRY: "false"
CHART_CACHE_DRIVER: "redis"
_REDIS_URL: "{{ template "harbor.redisForCore" . }}"
_REDIS_URL_REG: "{{ template "harbor.redisForGC" . }}"
PORTAL_URL: "http://{{ template "harbor.portal" . }}"
REGISTRYCTL_URL: "http://{{ template "harbor.registry" . }}:8080"
CLAIR_HEALTH_CHECK_SERVER_URL: "http://{{ template "harbor.clair" . }}:6061"
\ No newline at end of file
......@@ -19,7 +19,6 @@ spec:
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/core/core-cm.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
checksum/secret-jobservice: {{ include (print $.Template.BasePath "/jobservice/jobservice-secrets.yaml") . | sha256sum }}
{{- if .Values.core.podAnnotations }}
{{ toYaml .Values.core.podAnnotations | indent 8 }}
{{- end }}
......@@ -40,11 +39,6 @@ spec:
port: 8080
initialDelaySeconds: 20
periodSeconds: 10
envFrom:
- configMapRef:
name: "{{ template "harbor.core" . }}"
- secretRef:
name: "{{ template "harbor.core" . }}"
env:
- name: CORE_SECRET
valueFrom:
......@@ -56,6 +50,20 @@ spec:
secretKeyRef:
name: "{{ template "harbor.jobservice" . }}"
key: secret
- name: _REDIS_URL
value: {{ template "harbor.redisForCore" . }}
- name: _REDIS_URL_REG
value: {{ template "harbor.redisForGC" . }}
- name: LOG_LEVEL
value: {{ .Values.logLevel }}
- name: CONFIG_PATH
value: /etc/core/app.conf
- name: SYNC_REGISTRY
value: "false"
- name: ADMINSERVER_URL
value: "http://{{ template "harbor.adminserver" . }}"
- name: CHART_CACHE_DRIVER
value: "redis"
ports:
- containerPort: 8080
volumeMounts:
......@@ -69,18 +77,14 @@ spec:
mountPath: /etc/core/private_key.pem
subPath: tokenServicePrivateKey
- name: etc-localtime
mountPath: /etc/localtime
{{- if .Values.expose.tls.enabled }}
mountPath: /etc/localtime
{{- if eq (include "harbor.autoGenCert" .) "true" }}
- name: ca-download
mountPath: /etc/core/ca/ca.crt
subPath: ca.crt
{{- end }}
- name: psc
mountPath: /etc/core/token
{{- if .Values.core.resources }}
resources:
{{ toYaml .Values.core.resources | indent 10 }}
{{- end }}
volumes:
- name: etc-localtime
hostPath:
......@@ -96,15 +100,15 @@ spec:
path: key
- name: token-service-private-key
secret:
{{- if .Values.core.secretName }}
secretName: {{ .Values.core.secretName }}
{{- else }}
secretName: {{ template "harbor.core" . }}
{{- end }}
{{- if .Values.expose.tls.enabled }}
{{- if eq (include "harbor.autoGenCert" .) "true" }}
- name: ca-download
secret:
secretName: "{{ template "harbor.certificate-secret" . }}"
{{- if eq (include "harbor.autoGenCertForIngress" .) "true" }}
secretName: "{{ template "harbor.ingress" . }}"
{{- else }}
secretName: {{ template "harbor.nginx" . }}
{{- end }}
items:
- key: ca.crt
path: ca.crt
......@@ -122,4 +126,4 @@ spec:
{{- with .Values.core.tolerations }}
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
\ No newline at end of file
{{- end }}
......@@ -8,13 +8,7 @@ metadata:
type: Opaque
data:
secretKey: {{ .Values.secretKey | b64enc | quote }}
secret: {{ .Values.core.secret | default (randAlphaNum 16) | b64enc | quote }}
{{- if not .Values.core.secretName }}
secret: {{ randAlphaNum 16 | b64enc | quote }}
tokenServiceRootCertBundle: {{ $cert.Cert | b64enc | quote }}
tokenServicePrivateKey: {{ $cert.Key | b64enc | quote }}
{{- end }}
HARBOR_ADMIN_PASSWORD: {{ .Values.harborAdminPassword | b64enc | quote }}
POSTGRESQL_PASSWORD: {{ template "harbor.database.encryptedPassword" . }}
{{ if .Values.clair.enabled }}
CLAIR_DB_PASSWORD: {{ template "harbor.database.encryptedPassword" . }}
{{ end }}
\ No newline at end of file
......@@ -5,9 +5,6 @@ metadata:
labels:
{{ include "harbor.labels" . | indent 4 }}
spec:
{{- if (eq .Values.expose.ingress.controller "gce") }}
type: NodePort
{{- end }}
ports:
- port: 80
targetPort: 8080
......
{{- if eq .Values.database.type "internal" -}}
{{- $database := .Values.persistence.persistentVolumeClaim.database -}}
apiVersion: apps/v1
kind: StatefulSet
metadata:
......@@ -28,7 +27,6 @@ spec:
initContainers:
- name: "remove-lost-found"
image: {{ .Values.database.internal.image.repository }}:{{ .Values.database.internal.image.tag }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
command: ["rm", "-Rf", "/var/lib/postgresql/data/lost+found"]
volumeMounts:
- name: database-data
......@@ -49,10 +47,8 @@ spec:
- /docker-healthcheck.sh
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.database.internal.resources }}
resources:
{{ toYaml .Values.database.internal.resources | indent 10 }}
{{- end }}
envFrom:
- secretRef:
name: "{{ template "harbor.database" . }}"
......@@ -60,7 +56,7 @@ spec:
- name: database-data
mountPath: /var/lib/postgresql/data
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
......
{{- if eq .Values.expose.type "ingress" }}
{{- $ingress := .Values.expose.ingress -}}
{{- $tls := .Values.expose.tls -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: "{{ template "harbor.ingress.core" . }}"
name: "{{ template "harbor.ingress" . }}"
labels:
{{ include "harbor.labels" . | indent 4 }}
annotations:
{{ toYaml $ingress.annotations | indent 4 }}
spec:
{{- if .Values.expose.tls.enabled }}
{{- if $tls.enabled }}
tls:
- secretName: {{ template "harbor.certificate-secret" . }}
{{- if $ingress.host }}
hosts:
- {{ $ingress.host }}
- hosts:
- {{ $ingress.hosts.core }}
{{- if $tls.secretName }}
secretName: {{ $tls.secretName }}
{{- else }}
secretName: "{{ template "harbor.ingress" . }}"
{{- end }}
{{- end }}
{{- if eq .Values.expose.ingress.controller "gce" }}
rules:
- http:
paths:
- path: /*
backend:
serviceName: {{ template "harbor.portal" . }}
servicePort: 80
- path: /api/*
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
- path: /service/*
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
- path: /v2/*
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
- path: /chartrepo/*
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
- path: /c/*
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
{{- if $ingress.host }}
host: {{ $ingress.host }}
{{- if .Values.notary.enabled }}
- hosts:
- {{ $ingress.hosts.notary }}
{{- if $tls.notarySecretName }}
secretName: {{ $tls.notarySecretName }}
{{- else if $tls.secretName }}
secretName: {{ $tls.secretName }}
{{- else }}
secretName: "{{ template "harbor.ingress" . }}"
{{- end }}
{{- else }}
{{- end }}
{{- end }}
rules:
- http:
- host: {{ $ingress.hosts.core }}
http:
paths:
- path: /
backend:
......@@ -76,8 +59,13 @@ spec:
backend:
serviceName: {{ template "harbor.core" . }}
servicePort: 80
{{- if $ingress.host }}
host: {{ $ingress.host }}
{{- end }}
{{- end }}
{{- if .Values.notary.enabled }}
- host: {{ $ingress.hosts.notary }}
http:
paths:
- path: /
backend:
serviceName: {{ template "harbor.notary-server" . }}
servicePort: 4443
{{- end }}
{{- end }}
\ No newline at end of file
{{- if eq (include "harbor.autoGenCertForIngress" .) "true" }}
{{- $ca := genCA "harbor-ca" 365 }}
{{- $cert := genSignedCert .Values.expose.ingress.hosts.core nil (list .Values.expose.ingress.hosts.core .Values.expose.ingress.hosts.notary) 365 $ca }}
apiVersion: v1
kind: Secret
metadata:
name: "{{ template "harbor.ingress" . }}"
labels:
{{ include "harbor.labels" . | indent 4 }}
type: kubernetes.io/tls
data:
tls.crt: {{ $cert.Cert | b64enc | quote }}
tls.key: {{ $cert.Key | b64enc | quote }}
ca.crt: {{ $ca.Cert | b64enc | quote }}
{{- end }}
\ No newline at end of file
......@@ -36,4 +36,5 @@ data:
#Loggers for the job service
loggers:
- name: "STD_OUTPUT"
level: {{ .Values.logLevel | upper }}
\ No newline at end of file
level: {{ .Values.logLevel | upper }}
admin_server: "http://{{ template "harbor.adminserver" . }}"
......@@ -19,7 +19,6 @@ spec:
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/jobservice/jobservice-cm.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/jobservice/jobservice-secrets.yaml") . | sha256sum }}
checksum/secret-core: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
{{- if .Values.jobservice.podAnnotations }}
{{ toYaml .Values.jobservice.podAnnotations | indent 8 }}
{{- end }}
......@@ -40,10 +39,8 @@ spec:
port: 8080
initialDelaySeconds: 20
periodSeconds: 10
{{- if .Values.jobservice.resources }}
resources:
{{ toYaml .Values.jobservice.resources | indent 10 }}
{{- end }}
env:
- name: CORE_SECRET
valueFrom:
......@@ -55,8 +52,8 @@ spec:
secretKeyRef:
name: "{{ template "harbor.jobservice" . }}"
key: secret
- name: CORE_URL
value: "http://{{ template "harbor.core" . }}"
- name: ADMINSERVER_URL
value: "http://{{ template "harbor.adminserver" . }}"
- name: REGISTRY_CONTROLLER_URL
value: "http://{{ template "harbor.registry" . }}:8080"
- name: LOG_LEVEL
......@@ -70,7 +67,7 @@ spec:
- name: job-logs
mountPath: /var/log/jobs
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
......@@ -79,7 +76,12 @@ spec:
configMap:
name: "{{ template "harbor.jobservice" . }}"
- name: job-logs
{{- if and .Values.persistence.enabled (eq .Values.jobservice.jobLogger "file") }}
persistentVolumeClaim:
claimName: {{ .Values.persistence.persistentVolumeClaim.jobservice.existingClaim | default (include "harbor.jobservice" .) }}
{{- else }}
emptyDir: {}
{{- end }}
{{- with .Values.jobservice.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
......
......@@ -6,4 +6,4 @@ metadata:
{{ include "harbor.labels" . | indent 4 }}
type: Opaque
data:
secret: {{ .Values.jobservice.secret | default (randAlphaNum 16) | b64enc | quote }}
secret: {{ randAlphaNum 16 | b64enc | quote }}
\ No newline at end of file
......@@ -67,19 +67,6 @@ data:
proxy_buffering off;
proxy_request_buffering off;
}
location /chartrepo/ {
proxy_pass http://core/chartrepo/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_request_buffering off;
}
location /c/ {
proxy_pass http://core/c/;
......
......@@ -42,6 +42,41 @@ data:
access_log /dev/stdout timed_combined;
{{- if .Values.notary.enabled }}
server {
listen 4443 ssl;
server_tokens off;
# ssl
ssl_certificate /etc/nginx/cert/tls.crt;
ssl_certificate_key /etc/nginx/cert/tls.key;
# recommendations from https://raymii.org/s/tutorials/strong_ssl_security_on_nginx.html
ssl_protocols tlsv1.1 tlsv1.2;
ssl_ciphers '!aNULL:kECDH+AESGCM:ECDH+AESGCM:RSA+AESGCM:kECDH+AES:ECDH+AES:RSA+AES:';
ssl_prefer_server_ciphers on;
ssl_session_cache shared:ssl:10m;
# disable any limits to avoid http 413 for large image uploads
client_max_body_size 0;
# required to avoid http 411: see issue #1486 (https://github.com/docker/docker/issues/1486)
chunked_transfer_encoding on;
location /v2/ {
proxy_pass http://notary-server/v2/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_request_buffering off;
}
}
{{- end }}
server {
listen 443 ssl;
# server_name harbordomain.com;
......@@ -90,19 +125,6 @@ data:
proxy_buffering off;
proxy_request_buffering off;
}
location /chartrepo/ {
proxy_pass http://core/chartrepo/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_request_buffering off;
}
location /c/ {
proxy_pass http://core/c/;
......@@ -149,22 +171,6 @@ data:
location /service/notifications {
return 404;
}
{{- if .Values.notary.enabled }}
location /notary/ {
proxy_pass http://notary-server;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_request_buffering off;
rewrite /notary/(.*) /$1 break;
}
{{- end }}
}
server {
listen 80;
......@@ -172,4 +178,5 @@ data:
return 301 https://$host$request_uri;
}
}
{{- end }}
\ No newline at end of file
{{- end }}
\ No newline at end of file
......@@ -23,8 +23,8 @@ spec:
{{- else }}
checksum/configmap: {{ include (print $.Template.BasePath "/nginx/configmap-https.yaml") . | sha256sum }}
{{- end }}
{{- if eq (include "harbor.autoGenCert" .) "true" }}
checksum/secret: {{ include (print $.Template.BasePath "/common/certificate-secret.yaml") . | sha256sum }}
{{- if eq (include "harbor.autoGenCertForNginx" .) "true" }}
checksum/secret: {{ include (print $.Template.BasePath "/nginx/secret.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.nginx.podAnnotations }}
{{ toYaml .Values.nginx.podAnnotations | indent 8 }}
......@@ -46,13 +46,12 @@ spec:
port: 80
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.nginx.resources }}
resources:
{{ toYaml .Values.nginx.resources | indent 10 }}
{{- end }}
ports:
- containerPort: 80
- containerPort: 443
- containerPort: 4443
volumeMounts:
- name: config
mountPath: /etc/nginx/nginx.conf
......@@ -68,7 +67,11 @@ spec:
{{- if .Values.expose.tls.enabled }}
- name: certificate
secret:
secretName: {{ template "harbor.certificate-secret" . }}
{{- if .Values.expose.tls.secretName }}
secretName: {{ .Values.expose.tls.secretName }}
{{- else }}
secretName: {{ template "harbor.nginx" . }}
{{- end }}
{{- end }}
{{- with .Values.nginx.nodeSelector }}
nodeSelector:
......
{{- if eq (include "harbor.autoGenCertForNginx" .) "true" }}
{{- $ca := genCA "harbor-ca" 365 }}
{{- $cn := (required "The \"expose.tls.commonName\" is required!" .Values.expose.tls.commonName) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "harbor.nginx" . }}
labels:
{{ include "harbor.labels" . | indent 4 }}
type: Opaque
data:
{{- if regexMatch `^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$` $cn }}
{{- $cert := genSignedCert $cn (list $cn) nil 365 $ca }}
tls.crt: {{ $cert.Cert | b64enc | quote }}
tls.key: {{ $cert.Key | b64enc | quote }}
ca.crt: {{ $ca.Cert | b64enc | quote }}
{{- else }}
{{- $cert := genSignedCert $cn nil (list $cn) 365 $ca }}
tls.crt: {{ $cert.Cert | b64enc | quote }}
tls.key: {{ $cert.Key | b64enc | quote }}
ca.crt: {{ $ca.Cert | b64enc | quote }}
{{- end }}
{{- end }}
\ No newline at end of file
......@@ -5,19 +5,25 @@ metadata:
{{- if eq .Values.expose.type "clusterIP" }}
{{- $clusterIP := .Values.expose.clusterIP }}
name: {{ $clusterIP.name }}
labels:
{{ include "harbor.labels" . | indent 4 }}
spec:
type: ClusterIP
ports:
- name: http
port: {{ $clusterIP.ports.http }}
port: {{ $clusterIP.ports.httpPort }}
targetPort: 80
{{- if .Values.expose.tls.enabled }}
- name: https
port: {{ $clusterIP.ports.https }}
port: {{ $clusterIP.ports.httpsPort }}
targetPort: 443
{{- end }}
{{- if .Values.notary.enabled }}
- name: notary
port: {{ $clusterIP.ports.notaryPort }}
targetPort: 4443
{{- end }}
{{- else if eq .Values.expose.type "nodePort" }}
{{- $nodePort := .Values.expose.nodePort }}
name: {{ $nodePort.name }}
......@@ -40,21 +46,13 @@ spec:
nodePort: {{ $nodePort.ports.https.nodePort }}
{{- end }}
{{- end }}
{{- else if eq .Values.expose.type "loadBalancer" }}
{{- $loadBalancer := .Values.expose.loadBalancer }}
name: {{ $loadBalancer.name }}
labels:
{{ include "harbor.labels" . | indent 4 }}
spec:
type: LoadBalancer
ports:
- name: http
port: {{ $loadBalancer.ports.http }}
targetPort: 80
{{- if .Values.expose.tls.enabled }}
- name: https
port: {{ $loadBalancer.ports.https }}
targetPort: 443
{{- if .Values.notary.enabled }}
- name: notary
port: {{ $nodePort.ports.notary.port }}
targetPort: 4443
{{- if $nodePort.ports.notary.nodePort }}
nodePort: {{ $nodePort.ports.notary.nodePort }}
{{- end }}
{{- end }}
{{- end }}
selector:
......
......@@ -9,14 +9,12 @@ metadata:
data:
{{ $ca := genCA "harbor-notary-ca" 365 }}
{{ $cert := genSignedCert (include "harbor.notary-signer" .) nil nil 365 $ca }}
{{- if not .Values.notary.secretName }}
notary-signer-ca.crt: |
{{ $ca.Cert | indent 4 }}
{{ $ca.Cert | indent 4 }}
notary-signer.crt: |
{{ $cert.Cert | indent 4 }}
notary-signer.key: |
{{ $cert.Key | indent 4 }}
{{- end }}
{{ $cert.Key | indent 4 }}
server-config.postgres.json: |
{
"server": {
......@@ -26,11 +24,7 @@ data:
"type": "remote",
"hostname": "{{ template "harbor.notary-signer" . }}",
"port": "7899",
{{- if not .Values.notary.secretName }}
"tls_ca_file": "./notary-signer-ca.crt",
{{- else }}
"tls_ca_file": "/etc/ssl/notary/cert/notary-signer-ca.crt",
{{- end }}
"key_algorithm": "ecdsa"
},
"logging": {
......@@ -54,13 +48,8 @@ data:
{
"server": {
"grpc_addr": ":7899",
{{- if not .Values.notary.secretName }}
"tls_cert_file": "./notary-signer.crt",
"tls_key_file": "./notary-signer.key"
{{- else }}
"tls_cert_file": "/etc/ssl/notary/cert/notary-signer.crt",
"tls_key_file": "/etc/ssl/notary/cert/notary-signer.key"
{{- end }}
},
"logging": {
"level": "{{ .Values.logLevel }}"
......
......@@ -19,7 +19,6 @@ spec:
app: notary-server
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/notary/notary-cm.yaml") . | sha256sum }}
checksum/secret-core: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
{{- if .Values.notary.podAnnotations }}
{{ toYaml .Values.notary.podAnnotations | indent 8 }}
{{- end }}
......@@ -28,10 +27,8 @@ spec:
- name: notary-server
image: {{ .Values.notary.server.image.repository }}:{{ .Values.notary.server.image.tag }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
{{- if .Values.notary.server.resources }}
resources:
{{ toYaml .Values.notary.server.resources | indent 10 }}
{{- end }}
env:
- name: MIGRATIONS_PATH
value: migrations/server/postgresql
......@@ -41,15 +38,10 @@ spec:
- name: notary-config
mountPath: /etc/notary
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
- name: root-certificate
mountPath: /root.crt
subPath: tokenServiceRootCertBundle
{{- if .Values.notary.secretName }}
- name: notary-ca
mountPath: /etc/ssl/notary/cert/notary-signer-ca.crt
subPath: ca
{{- end }}
volumes:
- name: etc-localtime
hostPath:
......@@ -59,16 +51,7 @@ spec:
name: "{{ template "harbor.notary-server" . }}"
- name: root-certificate
secret:
{{- if .Values.core.secretName }}
secretName: {{ .Values.core.secretName }}
{{- else }}
secretName: {{ template "harbor.core" . }}
{{- end }}
{{- if .Values.notary.secretName }}
- name: notary-ca
secret:
secretName: {{ .Values.notary.secretName }}
{{- end }}
{{- with .Values.notary.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
......
......@@ -24,10 +24,8 @@ spec:
- name: notary-signer
image: {{ .Values.notary.signer.image.repository }}:{{ .Values.notary.signer.image.tag }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
{{- if .Values.notary.signer.resources }}
resources:
{{ toYaml .Values.notary.signer.resources | indent 10 }}
{{- end }}
env:
- name: MIGRATIONS_PATH
value: migrations/signer/postgresql
......@@ -39,18 +37,7 @@ spec:
- name: notary-config
mountPath: /etc/notary
- name: etc-localtime
mountPath: /etc/localtime
{{- if .Values.notary.secretName }}
- name: notary-cert
mountPath: /etc/ssl/notary/cert/notary-signer-ca.crt
subPath: ca
- name: notary-cert
mountPath: /etc/ssl/notary/cert/notary-signer.crt
subPath: crt
- name: notary-cert
mountPath: /etc/ssl/notary/cert/notary-signer.key
subPath: key
{{- end }}
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
......@@ -58,11 +45,6 @@ spec:
- name: notary-config
configMap:
name: "{{ template "harbor.notary-server" . }}"
{{- if .Values.notary.secretName }}
- name: notary-cert
secret:
secretName: {{ .Values.notary.secretName }}
{{- end }}
{{- with .Values.notary.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
......
......@@ -6,9 +6,6 @@ metadata:
labels:
{{ include "harbor.labels" . | indent 4 }}
spec:
{{- if (eq .Values.expose.ingress.controller "gce") }}
type: NodePort
{{- end }}
ports:
- port: 4443
selector:
......
......@@ -49,7 +49,7 @@ spec:
volumes:
- name: etc-localtime
hostPath:
path: /etc/localtime
path: /etc/localtime
{{- with .Values.portal.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
......
......@@ -5,9 +5,6 @@ metadata:
labels:
{{ include "harbor.labels" . | indent 4 }}
spec:
{{- if (eq .Values.expose.ingress.controller "gce") }}
type: NodePort
{{- end }}
ports:
- port: 80
targetPort: 80
......
......@@ -37,19 +37,17 @@ spec:
port: 6379
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.redis.internal.resources }}
resources:
{{ toYaml .Values.redis.internal.resources | indent 10 }}
{{- end }}
volumeMounts:
- name: data
mountPath: /var/lib/redis
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
volumes:
- name: etc-localtime
hostPath:
path: /etc/localtime
path: /etc/localtime
- name: data
emptyDir: {}
{{- with .Values.redis.internal.nodeSelector }}
......@@ -64,4 +62,4 @@ spec:
tolerations:
{{ toYaml . | indent 8 }}
{{- end }}
{{- end -}}
{{- end -}}
......@@ -21,10 +21,9 @@ data:
enabled: false
delete:
enabled: true
redirect:
disable: false
redis:
addr: "{{ template "harbor.redis.host" . }}:{{ template "harbor.redis.port" . }}"
password: {{ template "harbor.redis.rawPassword" . }}
db: {{ template "harbor.redis.registryDatabaseIndex" . }}
http:
addr: :5000
......@@ -38,8 +37,6 @@ data:
realm: "{{ .Values.externalURL }}/service/token"
rootcertbundle: /etc/registry/root.crt
service: harbor-registry
validation:
disabled: true
notifications:
endpoints:
- name: harbor
......
......@@ -19,8 +19,6 @@ spec:
annotations:
checksum/configmap: {{ include (print $.Template.BasePath "/registry/registry-cm.yaml") . | sha256sum }}
checksum/secret: {{ include (print $.Template.BasePath "/registry/registry-secret.yaml") . | sha256sum }}
checksum/secret-jobservice: {{ include (print $.Template.BasePath "/jobservice/jobservice-secrets.yaml") . | sha256sum }}
checksum/secret-core: {{ include (print $.Template.BasePath "/core/core-secret.yaml") . | sha256sum }}
{{- if .Values.registry.podAnnotations }}
{{ toYaml .Values.registry.podAnnotations | indent 8 }}
{{- end }}
......@@ -41,10 +39,8 @@ spec:
port: 5000
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.registry.registry.resources }}
resources:
{{ toYaml .Values.registry.registry.resources | indent 10 }}
{{- end }}
{{ toYaml .Values.registry.resources | indent 10 }}
args: ["serve", "/etc/registry/config.yml"]
envFrom:
- secretRef:
......@@ -62,7 +58,7 @@ spec:
mountPath: /etc/registry/config.yml
subPath: config.yml
- name: etc-localtime
mountPath: /etc/localtime
mountPath: /etc/localtime
- name: registryctl
image: {{ .Values.registry.controller.image.repository }}:{{ .Values.registry.controller.image.tag }}
imagePullPolicy: {{ .Values.imagePullPolicy }}
......@@ -78,10 +74,8 @@ spec:
port: 8080
initialDelaySeconds: 1
periodSeconds: 10
{{- if .Values.registry.controller.resources }}
resources:
{{ toYaml .Values.registry.controller.resources | indent 10 }}
{{- end }}
{{ toYaml .Values.registry.resources | indent 10 }}
args: ["serve", "/etc/registry/config.yml"]
envFrom:
- secretRef:
......@@ -95,7 +89,7 @@ spec:
- name: JOBSERVICE_SECRET
valueFrom:
secretKeyRef:
name: {{ template "harbor.jobservice" . }}
name: "{{ template "harbor.fullname" . }}-jobservice"
key: secret
ports:
- containerPort: 8080
......@@ -113,14 +107,10 @@ spec:
volumes:
- name: etc-localtime
hostPath:
path: /etc/localtime
path: /etc/localtime
- name: registry-root-certificate
secret:
{{- if .Values.core.secretName }}
secretName: {{ .Values.core.secretName }}
{{- else }}
secretName: {{ template "harbor.core" . }}
{{- end }}
- name: registry-config
configMap:
name: "{{ template "harbor.registry" . }}"
......@@ -130,7 +120,7 @@ spec:
path: {{ .Values.registry.hostPath }}
{{- else }}
path: /data/{{ .Release.Namespace }}/{{ .Release.Name }}/registry
{{- end }}
{{- end }}
{{- with .Values.registry.nodeSelector }}
nodeSelector:
{{ toYaml . | indent 8 }}
......
......@@ -6,14 +6,11 @@ metadata:
{{ include "harbor.labels" . | indent 4 }}
type: Opaque
data:
REGISTRY_HTTP_SECRET: {{ .Values.registry.secret | default (randAlphaNum 16) | b64enc | quote }}
REGISTRY_REDIS_PASSWORD: {{ (include "harbor.redis.rawPassword" .) | b64enc | quote }}
REGISTRY_HTTP_SECRET: {{ randAlphaNum 16 | b64enc | quote }}
{{- $storage := .Values.persistence.imageChartStorage }}
{{- $type := $storage.type }}
{{- if eq $type "azure" }}
REGISTRY_STORAGE_AZURE_ACCOUNTKEY: {{ $storage.azure.accountkey | b64enc | quote }}
{{- else if eq $type "gcs" }}
GCS_KEY_DATA: {{ $storage.gcs.encodedkey | quote }}
{{- else if eq $type "s3" }}
{{- if $storage.s3.accesskey }}
REGISTRY_STORAGE_S3_ACCESSKEY: {{ $storage.s3.accesskey | b64enc | quote }}
......@@ -31,4 +28,4 @@ data:
{{- end }}
{{- else if eq $type "oss" }}
REGISTRY_STORAGE_OSS_ACCESSKEYSECRET: {{ $storage.oss.accesskeysecret | b64enc | quote }}
{{- end }}
{{- end }}
\ No newline at end of file
......@@ -3,11 +3,11 @@ expose:
tls:
enabled: false
ingress:
host: hub.wodcloud.local
controller: default
hosts:
core: hub.wodcloud.local
notary: notary.wodcloud.local
annotations:
ingress.kubernetes.io/ssl-redirect: "true"
rewriteAnnotation: traefik.ingress.kubernetes.io/rewrite-target
externalURL: https://hub.wodcloud.local
......@@ -32,6 +32,12 @@ core:
tag: v1.7.5
replicas: 1
adminserver:
image:
repository: registry.cn-qingdao.aliyuncs.com/wod/harbor-adminserver
tag: v1.7.5
replicas: 1
jobservice:
image:
repository: registry.cn-qingdao.aliyuncs.com/wod/harbor-jobservice
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment