diff --git a/.env b/.env old mode 100644 new mode 100755 index fc4b6bd..e45932a --- a/.env +++ b/.env @@ -1,8 +1,8 @@ -# master: 2023-12-11 +# master: 2023-12-13 # edited: 2023-12-11 # updated: 2023-12-11 -VM_VERSIONNING="2023-12-11" +VM_VERSIONNING="2023-12-13" diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 94c2d5a..a707e26 --- a/README.md +++ b/README.md @@ -2,19 +2,33 @@ **Try to setup Harbor under Docker with a all-in-one docker-compose.** +## References + +* [Good french posts](https://blog.stephane-robert.info/docs/developper/artefacts/harbor/) + +## Changelog + +### 2023-12-13 + +* **Switched** from _Bitnami_ docker images to _officials_ +* Use **official** _Harbor installer_ to grab files +* **Reworked** _docker-compose.yml_ and _folders_ structure +* **Tuned** _docker_compose_ to extends from my base docker-compose definitions +* Use **ssl/build.sh** to generate SSL self-signed certificates for Nginx + +### 2023-12-11 + +* Initial try with Bitnami docker images + Setup from : https://hub.docker.com/r/bitnami/harbor-core _Like this:_ -``` +``` bash curl -LO https://raw.githubusercontent.com/bitnami/containers/main/bitnami/harbor-portal/docker-compose.yml curl -L https://github.com/bitnami/containers/archive/main.tar.gz | tar xz --strip=2 containers-main/bitnami/harbor-portal && cp -RL harbor-portal/config . && rm -rf harbor-portal docker-compose up ``` Update docker-compose.yml with this repository one's. - -Good french posts: - -* https://blog.stephane-robert.info/docs/developper/artefacts/harbor/ diff --git a/_vm/common.yml b/_vm/common.yml old mode 100644 new mode 100755 index 52b5689..1a96e0f --- a/_vm/common.yml +++ b/_vm/common.yml @@ -1,7 +1,7 @@ version: "3.0" # -# updated: 2023-12-10 +# updated: 2023-12-13 # stack: x-common # diff --git a/_vm/host.yml b/_vm/host.yml old mode 100644 new mode 100755 index 7fca0de..e82a0e6 --- a/_vm/host.yml +++ b/_vm/host.yml @@ -1,7 +1,7 @@ version: "3.0" # -# updated: 2023-12-11 +# updated: 2023-12-13 # stack: host # diff --git a/conf/common/env b/conf/common/env new file mode 100755 index 0000000..48ff544 --- /dev/null +++ b/conf/common/env @@ -0,0 +1,6 @@ +CORE_SECRET=CHANGEME +JOBSERVICE_SECRET=CHANGEME +REGISTRY_CREDENTIAL_PASSWORD=CHANGEME +CSRF_KEY=CHANGEME +DATABASE_PASSWORD=CHANGEME +ADMIN_PASSWORD=CHANGEME diff --git a/conf/core/app.conf b/conf/core/app.conf old mode 100644 new mode 100755 index 6110364..28351cd --- a/conf/core/app.conf +++ b/conf/core/app.conf @@ -1,6 +1,6 @@ appname = Harbor -runmode = dev +runmode = prod enablegzip = true -[dev] +[prod] httpport = 8080 diff --git a/conf/core/certificates/.placeholder b/conf/core/certificates/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/conf/core/env b/conf/core/env new file mode 100755 index 0000000..2cf0703 --- /dev/null +++ b/conf/core/env @@ -0,0 +1,64 @@ +CONFIG_PATH=/etc/core/app.conf +UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem + +_REDIS_URL_CORE=redis://harbor-redis:6379?idle_timeout_seconds=30 +SYNC_QUOTA=true +CHART_CACHE_DRIVER=redis +_REDIS_URL_REG=redis://harbor-redis:6379/1?idle_timeout_seconds=30 + +LOG_LEVEL=info + +EXT_ENDPOINT=https://jimmu.home + +DATABASE_TYPE=postgresql +POSTGRESQL_HOST=harbor-postgresql +POSTGRESQL_PORT=5432 +POSTGRESQL_USERNAME=postgres +POSTGRESQL_PASSWORD=$DATABASE_PASSWORD +POSTGRESQL_DATABASE=registry +POSTGRESQL_SSLMODE=disable +POSTGRESQL_MAX_IDLE_CONNS=100 +POSTGRESQL_MAX_OPEN_CONNS=900 +POSTGRESQL_CONN_MAX_LIFETIME=5m +POSTGRESQL_CONN_MAX_IDLE_TIME=0 + +REGISTRY_URL=http://harbor-registry:5000 +PORTAL_URL=http://harbor-portal:8080 +TOKEN_SERVICE_URL=http://harbor-core:8080/service/token +CORE_URL=http://harbor-core:8080 +CORE_LOCAL_URL=http://127.0.0.1:8080 +JOBSERVICE_URL=http://harbor-jobservice:8080 +TRIVY_ADAPTER_URL=http://harbor-trivy-adapter:8080 +NOTARY_URL=http://harbor-notary-server:4443 +CHART_REPOSITORY_URL=http://harbor-chartmuseum:9999 +REGISTRY_CONTROLLER_URL=http://harbor-registryctl:8080 + +HARBOR_ADMIN_PASSWORD=$ADMIN_PASSWORD + +MAX_JOB_WORKERS=10 + +WITH_NOTARY=False +WITH_TRIVY=True +WITH_CHARTMUSEUM=False + +REGISTRY_STORAGE_PROVIDER_NAME=filesystem +READ_ONLY=false +RELOAD_KEY= +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user + +PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE=docker-hub,harbor,azure-acr,aws-ecr,google-gcr,quay,docker-registry,github-ghcr,jfrog-artifactory + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=registry,localhost,db,jobservice,redis,127.0.0.1,notary-server,exporter,portal,.local,registryctl,core,notary-signer,log,.internal,nginx,postgresql,chartmuseum,trivy-adapter + +PORT=8080 + +METRIC_ENABLE=true +METRIC_PATH=/metrics +METRIC_PORT=9090 +METRIC_NAMESPACE=harbor +METRIC_SUBSYSTEM=core + +CACHE_ENABLED=true +CACHE_EXPIRE_HOURS=24 diff --git a/conf/core/private_key.pem b/conf/core/private_key.pem deleted file mode 100644 index d049fd5..0000000 --- a/conf/core/private_key.pem +++ /dev/null @@ -1 +0,0 @@ -CHANGEME \ No newline at end of file diff --git a/conf/db/env b/conf/db/env new file mode 100755 index 0000000..b678701 --- /dev/null +++ b/conf/db/env @@ -0,0 +1 @@ +POSTGRES_PASSWORD=$DATABASE_PASSWORD diff --git a/conf/exporter/env b/conf/exporter/env new file mode 100755 index 0000000..ec25f2b --- /dev/null +++ b/conf/exporter/env @@ -0,0 +1,30 @@ +LOG_LEVEL=info + +HARBOR_EXPORTER_PORT=8080 +HARBOR_EXPORTER_METRICS_PATH=/metrics +HARBOR_EXPORTER_METRICS_ENABLED=true +HARBOR_EXPORTER_MAX_REQUESTS=30 +HARBOR_EXPORTER_CACHE_TIME=23 +HARBOR_EXPORTER_CACHE_CLEAN_INTERVAL=14400 + +HARBOR_METRIC_NAMESPACE=harbor +HARBOR_METRIC_SUBSYSTEM=exporter + +HARBOR_SERVICE_HOST=harbor-core + +HARBOR_REDIS_URL=redis://harbor-redis:6379/2?idle_timeout_seconds=30 +HARBOR_REDIS_NAMESPACE=harbor_job_service_namespace +HARBOR_REDIS_TIMEOUT=3600 +HARBOR_SERVICE_PORT=8080 +HARBOR_SERVICE_SCHEME=http + +HARBOR_DATABASE_HOST=harbor-postgresql +HARBOR_DATABASE_PORT=5432 +HARBOR_DATABASE_USERNAME=postgres +HARBOR_DATABASE_PASSWORD=$DATABASE_PASSWORD +HARBOR_DATABASE_DBNAME=registry +HARBOR_DATABASE_SSLMODE=disable +HARBOR_DATABASE_MAX_IDLE_CONNS=100 +HARBOR_DATABASE_MAX_OPEN_CONNS=900 +HARBOR_DATABASE_CONN_MAX_LIFETIME=5m +HARBOR_DATABASE_CONN_MAX_IDLE_TIME=0 diff --git a/conf/jobservice/config.yml b/conf/jobservice/config.yml old mode 100644 new mode 100755 index 0440756..f842fab --- a/conf/jobservice/config.yml +++ b/conf/jobservice/config.yml @@ -2,11 +2,6 @@ #Protocol used to serve protocol: "http" -#Config certification if use 'https' protocol -#https_config: -# cert: "server.crt" -# key: "server.key" - #Server listening port port: 8080 @@ -18,8 +13,9 @@ worker_pool: #Additional config if use 'redis' backend redis_pool: #redis://[arbitrary_username:password@]ipaddress:port/database_index - redis_url: redis://harbor-redis:6379/2 + redis_url: redis://harbor-redis:6379/2?idle_timeout_seconds=30 namespace: "harbor_job_service_namespace" + idle_timeout_second: 3600 #Loggers for the running job job_loggers: - name: "STD_OUTPUT" # logger backend name, only support "FILE" and "STD_OUTPUT" @@ -37,5 +33,8 @@ job_loggers: loggers: - name: "STD_OUTPUT" # Same with above level: "INFO" -#Admin server endpoint -admin_server: "http://adminserver:8080/" + +metric: + enabled: true + path: /metrics + port: 9090 diff --git a/conf/jobservice/env b/conf/jobservice/env new file mode 100755 index 0000000..f78c585 --- /dev/null +++ b/conf/jobservice/env @@ -0,0 +1,17 @@ +REGISTRY_URL=http://harbor-registry:5000 +CORE_URL=http://harbor-core:8080 +REGISTRY_CONTROLLER_URL=http://harbor-registryctl:8080 +_REDIS_URL_CORE=redis://harbor-redis:6379?idle_timeout_seconds=30 + +JOBSERVICE_WEBHOOK_JOB_MAX_RETRY=10 + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=harbor-redis,harbor-jobservice,harbor-registry,harbor-db,harbor-exporter,harbor-registryctl,harbor-core,harbor-portal,harbor-postgresql,harbor-trivy-adapter,harbor-nginx,harbor-proxy,registry,localhost,db,jobservice,redis,127.0.0.1,notary-server,exporter,portal,.local,registryctl,core,notary-signer,log,.internal,nginx,postgresql,chartmuseum,trivy-adapter +REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user + +METRIC_NAMESPACE=harbor +METRIC_SUBSYSTEM=jobservice + +CACHE_ENABLED=true +CACHE_EXPIRE_HOURS=24 diff --git a/conf/log/logrotate.conf b/conf/log/logrotate.conf new file mode 100755 index 0000000..97f5f93 --- /dev/null +++ b/conf/log/logrotate.conf @@ -0,0 +1,8 @@ +/var/log/docker/*.log { + rotate 50 + size 200M + copytruncate + compress + missingok + nodateext +} \ No newline at end of file diff --git a/conf/log/rsyslog_docker.conf b/conf/log/rsyslog_docker.conf new file mode 100755 index 0000000..0be27a6 --- /dev/null +++ b/conf/log/rsyslog_docker.conf @@ -0,0 +1,7 @@ +# Rsyslog configuration file for docker. + +template(name="DynaFile" type="string" string="/var/log/docker/%programname%.log") + +if $programname != "rsyslogd" then { + action(type="omfile" dynaFile="DynaFile") +} \ No newline at end of file diff --git a/conf/nginx/conf.d/.placeholder b/conf/nginx/conf.d/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/conf/nginx/nginx.conf b/conf/nginx/nginx.conf new file mode 100755 index 0000000..8e8d2e4 --- /dev/null +++ b/conf/nginx/nginx.conf @@ -0,0 +1,187 @@ +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 3096; + use epoll; + multi_accept on; +} + +http { + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + tcp_nodelay on; + include /etc/nginx/conf.d/*.upstream.conf; + + # this is necessary for us to be able to disable request buffering in all cases + proxy_http_version 1.1; + + upstream core { + server harbor-core:8080; + } + + upstream portal { + server harbor-portal:8080; + } + + log_format timed_combined '$remote_addr - ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent" ' + '$request_time $upstream_response_time $pipe'; + + access_log /dev/stdout timed_combined; + + map $http_x_forwarded_proto $x_forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; + } + + include /etc/nginx/conf.d/*.server.conf; + + server { + listen 8443 ssl; +# server_name harbordomain.com; + server_tokens off; + # SSL + ssl_certificate /etc/cert/server.crt; + ssl_certificate_key /etc/cert/server.key; + + # Recommendations from https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html + ssl_protocols TLSv1.2; + ssl_ciphers '!aNULL:kECDH+AESGCM:ECDH+AESGCM:RSA+AESGCM:kECDH+AES:ECDH+AES:RSA+AES:'; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) + chunked_transfer_encoding on; + + # Add extra headers + add_header Strict-Transport-Security "max-age=31536000; includeSubdomains; preload"; + add_header X-Frame-Options DENY; + add_header Content-Security-Policy "frame-ancestors 'none'"; + + # customized location config file can place to /etc/nginx dir with prefix harbor.https. and suffix .conf + include /etc/nginx/conf.d/harbor.https.*.conf; + + location / { + proxy_pass http://portal/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_cookie_path / "/; HttpOnly; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /c/ { + proxy_pass http://core/c/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /api/ { + proxy_pass http://core/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /chartrepo/ { + proxy_pass http://core/chartrepo/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /v1/ { + return 404; + } + + location /v2/ { + proxy_pass http://core/v2/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + proxy_buffering off; + proxy_request_buffering off; + proxy_send_timeout 900; + proxy_read_timeout 900; + } + + location /service/ { + proxy_pass http://core/service/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $x_forwarded_proto; + + proxy_cookie_path / "/; Secure"; + + proxy_buffering off; + proxy_request_buffering off; + } + + location /service/notifications { + return 404; + } + } + server { + listen 8080; + #server_name harbordomain.com; + return 308 https://$host:443$request_uri; + } + upstream core_metrics { + server harbor-core:9090; + } + + upstream js_metrics { + server harbor-jobservice:9090; + } + + upstream registry_metrics { + server harbor-registry:9090; + } + + upstream harbor_exporter { + server harbor-exporter:8080; + } + + server { + listen 9090; + location = /metrics { + if ($arg_comp = core) { proxy_pass http://core_metrics; } + if ($arg_comp = jobservice) { proxy_pass http://js_metrics; } + if ($arg_comp = registry) { proxy_pass http://registry_metrics; } + proxy_pass http://harbor_exporter; + } + } +} diff --git a/conf/portal/nginx.conf b/conf/portal/nginx.conf new file mode 100755 index 0000000..3058d65 --- /dev/null +++ b/conf/portal/nginx.conf @@ -0,0 +1,42 @@ + +worker_processes auto; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + server { + listen 8080; + server_name localhost; + + root /usr/share/nginx/html; + index index.html index.htm; + include /etc/nginx/mime.types; + + gzip on; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + location /devcenter-api-2.0 { + try_files $uri $uri/ /swagger-ui-index.html; + } + + location / { + try_files $uri $uri/ /index.html; + } + + location = /index.html { + add_header Cache-Control "no-store, no-cache, must-revalidate"; + } + } +} \ No newline at end of file diff --git a/conf/proxy/nginx.conf b/conf/proxy/nginx.conf deleted file mode 100644 index bffc1a4..0000000 --- a/conf/proxy/nginx.conf +++ /dev/null @@ -1,130 +0,0 @@ -worker_processes auto; -error_log "/opt/bitnami/nginx/logs/error.log"; -pid "/opt/bitnami/nginx/tmp/nginx.pid"; - -events { - worker_connections 1024; - use epoll; - multi_accept on; -} - -http { - tcp_nodelay on; - - # this is necessary for us to be able to disable request buffering in all cases - proxy_http_version 1.1; - - upstream core { - server harbor-core:8080; - } - - upstream portal { - server harbor-portal:8080; - } - - log_format timed_combined '$remote_addr - ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '$request_time $upstream_response_time $pipe'; - - client_body_temp_path "/opt/bitnami/nginx/tmp/client_body" 1 2; - proxy_temp_path "/opt/bitnami/nginx/tmp/proxy" 1 2; - fastcgi_temp_path "/opt/bitnami/nginx/tmp/fastcgi" 1 2; - scgi_temp_path "/opt/bitnami/nginx/tmp/scgi" 1 2; - uwsgi_temp_path "/opt/bitnami/nginx/tmp/uwsgi" 1 2; - - server { - listen 8080; - server_tokens off; - # disable any limits to avoid HTTP 413 for large image uploads - client_max_body_size 0; - - # costumized location config file can place to /opt/bitnami/nginx/conf with prefix harbor.http. and suffix .conf - include /opt/bitnami/conf/nginx/conf.d/harbor.http.*.conf; - - location / { - proxy_pass http://portal/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /c/ { - proxy_pass http://core/c/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /api/ { - proxy_pass http://core/api/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /chartrepo/ { - proxy_pass http://core/chartrepo/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /v1/ { - return 404; - } - - location /v2/ { - proxy_pass http://core/v2/; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - proxy_buffering off; - proxy_request_buffering off; - } - - location /service/ { - proxy_pass http://core/service/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /service/notifications { - return 404; - } - } -} diff --git a/conf/registry/config.yml b/conf/registry/config.yml old mode 100644 new mode 100755 index 71daa4c..2c08de2 --- a/conf/registry/config.yml +++ b/conf/registry/config.yml @@ -10,27 +10,37 @@ storage: rootdirectory: /storage maintenance: uploadpurging: - enabled: false + enabled: true + age: 168h + interval: 24h + dryrun: false delete: enabled: true redis: addr: harbor-redis:6379 + readtimeout: 10s + writetimeout: 10s + dialtimeout: 10s password: db: 1 + pool: + maxidle: 100 + maxactive: 500 + idletimeout: 60s http: addr: :5000 secret: placeholder debug: - addr: localhost:5001 + addr: :9090 + prometheus: + enabled: true + path: /metrics auth: htpasswd: realm: harbor-registry-basic-realm path: /etc/registry/passwd -notifications: - endpoints: - - name: harbor - disabled: false - url: http://harbor-core:8080/service/notifications - timeout: 3000ms - threshold: 5 - backoff: 1s +validation: + disabled: true +compatibility: + schema1: + enabled: true \ No newline at end of file diff --git a/conf/registry/passwd b/conf/registry/passwd old mode 100644 new mode 100755 index d049fd5..1c3d2d0 --- a/conf/registry/passwd +++ b/conf/registry/passwd @@ -1 +1 @@ -CHANGEME \ No newline at end of file +harbor_registry_user:$2y$05$tbzR8jC.JJGMbaEnoX8.N.wXsMhpKoIIRxpx81BQQKm5DmIblm9fa diff --git a/conf/registry/root.crt b/conf/registry/root.crt old mode 100644 new mode 100755 index d049fd5..e69de29 --- a/conf/registry/root.crt +++ b/conf/registry/root.crt @@ -1 +0,0 @@ -CHANGEME \ No newline at end of file diff --git a/conf/registryctl/config.yml b/conf/registryctl/config.yml old mode 100644 new mode 100755 index 636f674..2c70735 --- a/conf/registryctl/config.yml +++ b/conf/registryctl/config.yml @@ -1,9 +1,5 @@ --- protocol: "http" port: 8080 -log_level: "INFO" -registry_config: "/etc/registry/config.yml" - -#https_config: -# cert: "server.crt" -# key: "server.key" +log_level: info +registry_config: "/etc/registry/config.yml" \ No newline at end of file diff --git a/conf/registryctl/env b/conf/registryctl/env new file mode 100755 index 0000000..e69de29 diff --git a/conf/shared/trust-certificates/.placeholder b/conf/shared/trust-certificates/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/conf/trivy-adapter/env b/conf/trivy-adapter/env new file mode 100755 index 0000000..fa6d59d --- /dev/null +++ b/conf/trivy-adapter/env @@ -0,0 +1,24 @@ +SCANNER_LOG_LEVEL=info + +SCANNER_REDIS_URL=redis://harbor-redis:6379/5?idle_timeout_seconds=30 +SCANNER_STORE_REDIS_URL=redis://harbor-redis:6379/5?idle_timeout_seconds=30 +SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store + +SCANNER_JOB_QUEUE_REDIS_URL=redis://harbor-redis:6379/5?idle_timeout_seconds=30 +SCANNER_JOB_QUEUE_REDIS_NAMESPACE=harbor.scanner.trivy:job-queue + +SCANNER_TRIVY_CACHE_DIR=/home/scanner/.cache/trivy +SCANNER_TRIVY_REPORTS_DIR=/home/scanner/.cache/reports +SCANNER_TRIVY_VULN_TYPE=os,library +SCANNER_TRIVY_SEVERITY=UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL +SCANNER_TRIVY_IGNORE_UNFIXED=False +SCANNER_TRIVY_SKIP_UPDATE=False +SCANNER_TRIVY_OFFLINE_SCAN=False +SCANNER_TRIVY_SECURITY_CHECKS=vuln +SCANNER_TRIVY_GITHUB_TOKEN= +SCANNER_TRIVY_INSECURE=False +SCANNER_TRIVY_TIMEOUT=5m0s + +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=registry,localhost,db,jobservice,redis,127.0.0.1,notary-server,exporter,portal,.local,registryctl,core,notary-signer,log,.internal,nginx,postgresql,chartmuseum,trivy-adapter diff --git a/datas/.placeholder b/datas/.placeholder deleted file mode 100644 index d049fd5..0000000 --- a/datas/.placeholder +++ /dev/null @@ -1 +0,0 @@ -CHANGEME \ No newline at end of file diff --git a/datas/ca_download/.placeholder b/datas/ca_download/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/job_logs/.placeholder b/datas/job_logs/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/redis/.placeholder b/datas/redis/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/registry/.placeholder b/datas/registry/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/secret/cert/.placeholder b/datas/secret/cert/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/secret/core/.placeholder b/datas/secret/core/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/secret/keys/.placeholder b/datas/secret/keys/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/secret/registry/.placeholder b/datas/secret/registry/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/trivy-adapter/reports/.placeholder b/datas/trivy-adapter/reports/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/trivy-adapter/trivy/.placeholder b/datas/trivy-adapter/trivy/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/datas/var/log/harbor/.placeholder b/datas/var/log/harbor/.placeholder new file mode 100755 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml old mode 100644 new mode 100755 index 0e610cf..0ec56ae --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,26 @@ version: "3.0" # -# updated: 2023-12-12 +# updated: 2023-12-13 # stack: harbor # # -# Login: admin / bitnami +# Default login: +# admin / Harbor12345 +# +# To do after git clone, +# +# Fix rights for config files: +# find conf/ -type f -exec chmod 0755 \{\} \; +# +# Setup permissions (with root): +# chown -R root:root . +# chown -R 10000:10000 datas/ca_download +# chown -R 10000:10000 datas/job_logs +# chown -R 10000:10000 datas/registry +# chown -R 999:999 datas/database +# chown -R 999:999 datas/redis # networks: @@ -17,386 +31,441 @@ networks: services: - harbor-registry: + harbor-log: extends: file: ./_vm/common.yml service: x-common user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - - SETGID - CHOWN - - SYS_ADMIN - - MKNOD - - SYS_CHROOT - container_name: harbor-registry - hostname: harbor-registry - image: docker.io/bitnami/harbor-registry:2 + - DAC_OVERRIDE + - SETGID + - SETUID + container_name: harbor-log + hostname: harbor-log + image: goharbor/harbor-log:v2.7.4 restart: "no" + ports: + - 0.0.0.0:1514:10514 + networks: + - harbor + labels: + com.stack.name: "harbor" + com.stack.service.name: "log" + volumes: + - ./conf/log/logrotate.conf:/etc/logrotate.d/logrotate.conf:ro + - ./conf/log/rsyslog_docker.conf:/etc/rsyslog.d/rsyslog_docker.conf:ro + - ./datas/var/log/harbor/:/var/log/docker/:rw + deploy: + resources: + limits: + memory: 1G + pids: 8192 + + harbor-registry: + extends: + file: ./_vm/common.yml + service: x-common + user: 10000:10000 + cap_add: + - CHOWN + - SETGID + - SETUID + container_name: harbor-registry + hostname: harbor-registry + image: goharbor/registry-photon:v2.7.4 + restart: "no" + depends_on: + - harbor-log networks: - harbor - environment: - - REGISTRY_HTTP_SECRET=CHANGEME labels: com.stack.name: "harbor" com.stack.service.name: "registry" volumes: + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./conf/registry/:/etc/registry/:z + - ./datas/secret/registry/root.crt:/etc/registry/root.crt:ro - ./datas/registry:/storage:rw - - ./conf/registry:/etc/registry/:ro deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "registry" harbor-registryctl: extends: - file: ../_vm/common.yml - service: x-common + file: ./_vm/host.yml + service: x-host user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - SETGID + - SETUID + - DAC_OVERRIDE - CHOWN - SYS_ADMIN - MKNOD - - SYS_CHROOT + - DAC_READ_SEARCH container_name: harbor-registryctl hostname: harbor-registryctl - image: docker.io/bitnami/harbor-registryctl:2 + image: goharbor/harbor-registryctl:v2.7.4 restart: "no" + depends_on: + - harbor-log networks: - harbor - environment: - - CORE_SECRET=CHANGEME - - JOBSERVICE_SECRET=CHANGEME - - REGISTRY_HTTP_SECRET=CHANGEME + env_file: + - ./conf/common/env + - ./conf/registryctl/env labels: com.stack.name: "harbor" com.stack.service.name: "registryctl" volumes: - - ./conf/registry:/etc/registry/:ro - - ./conf/registryctl/config.yml:/etc/registryctl/config.yml:ro + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./conf/registry/:/etc/registry/:z + - ./conf/registryctl/config.yml:/etc/registryctl/config.yml:rw - ./datas/registry:/storage:rw deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "registryctl" harbor-postgresql: extends: - file: ../_vm/common.yml + file: ./_vm/common.yml service: x-common - user: 0:0 + user: 999:999 + privileged: true cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - - SETGID - CHOWN - - SYS_ADMIN - - MKNOD - - SYS_CHROOT + - DAC_OVERRIDE + - SETGID + - SETUID container_name: harbor-postgresql hostname: harbor-postgresql - image: docker.io/bitnami/postgresql:13 + image: goharbor/harbor-db:v2.7.4 restart: "no" + depends_on: + - harbor-log networks: - harbor - environment: - - POSTGRESQL_PASSWORD=bitnami - - POSTGRESQL_DATABASE=registry + env_file: + - ./conf/common/env + - ./conf/db/env labels: com.stack.name: "harbor" com.stack.service.name: "postgresql" volumes: - - ./datas/postgresql:/bitnami/postgresql:rw + - ./datas/database:/var/lib/postgresql/data:rw + tmpfs: + - /run/postgresql:rw,size=64M deploy: resources: limits: memory: 1G pids: 8192 + shm_size: '1gb' + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "postgresql" harbor-core: extends: - file: ../_vm/common.yml - service: x-common + file: ./_vm/host.yml + service: x-host user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - SETGID + - SETUID + - DAC_OVERRIDE - CHOWN - SYS_ADMIN - MKNOD - - SYS_CHROOT + - DAC_READ_SEARCH container_name: harbor-core hostname: harbor-core - image: docker.io/bitnami/harbor-core:2 + image: goharbor/harbor-core:v2.7.4 restart: "no" networks: - harbor depends_on: + - harbor-log - harbor-registry - environment: - - CORE_KEY=CHANGEME - - _REDIS_URL_CORE=redis://harbor-redis:6379/0 - - SYNC_REGISTRY=false - - CHART_CACHE_DRIVER=redis - - _REDIS_URL_REG=redis://harbor-redis:6379/1 - - PORT=8080 - - LOG_LEVEL=info - - EXT_ENDPOINT=http://0.0.0.0 - - DATABASE_TYPE=postgresql - - REGISTRY_CONTROLLER_URL=http://harbor-registryctl:8080 - - POSTGRESQL_HOST=harbor-postgresql - - POSTGRESQL_PORT=5432 - - POSTGRESQL_DATABASE=registry - - POSTGRESQL_USERNAME=postgres - - POSTGRESQL_PASSWORD=bitnami - - POSTGRESQL_SSLMODE=disable - - REGISTRY_URL=http://harbor-registry:5000 - - TOKEN_SERVICE_URL=http://harbor-core:8080/service/token - - HARBOR_ADMIN_PASSWORD=bitnami - - CORE_SECRET=CHANGEME - - JOBSERVICE_SECRET=CHANGEME - - ADMIRAL_URL= - - CORE_URL=http://harbor-core:8080 - - JOBSERVICE_URL=http://harbor-jobservice:8080 - - REGISTRY_STORAGE_PROVIDER_NAME=filesystem - - REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user - - REGISTRY_CREDENTIAL_PASSWORD=harbor_registry_password - - READ_ONLY=false - - RELOAD_KEY= + - harbor-redis + - harbor-postgresql + env_file: + - ./conf/common/env + - ./conf/core/env labels: com.stack.name: "harbor" com.stack.service.name: "core" volumes: + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./conf/core/certificates/:/etc/core/certificates/:z - ./conf/core/app.conf:/etc/core/app.conf:ro - - ./conf/core/private_key.pem:/etc/core/private_key.pem:ro - - ./datas/core:/data:rw + - ./datas/secret/core/private_key.pem:/etc/core/private_key.pem:ro + - ./datas/secret/keys/secretkey:/etc/core/key:ro + - ./datas/ca_download/:/etc/core/ca/:rw + - ./datas/:/data/:rw deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "core" harbor-portal: extends: - file: ../_vm/common.yml + file: ./_vm/common.yml service: x-common user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - - SETGID - CHOWN - - SYS_ADMIN - - MKNOD - - SYS_CHROOT + - SETGID + - SETUID + - NET_BIND_SERVICE container_name: harbor-portal hostname: harbor-portal - image: docker.io/bitnami/harbor-portal:2 + image: goharbor/harbor-portal:v2.7.4 restart: "no" networks: - harbor depends_on: - - harbor-core + - harbor-log labels: com.stack.name: "harbor" com.stack.service.name: "portal" + volumes: + - ./conf/portal/nginx.conf:/etc/nginx/nginx.conf:ro deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "portal" harbor-jobservice: extends: - file: ../_vm/common.yml - service: x-common - user: 0:0 + file: ./_vm/host.yml + service: x-host + user: 10000:10000 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - SETGID + - SETUID + - DAC_OVERRIDE - CHOWN - SYS_ADMIN - MKNOD - - SYS_CHROOT + - DAC_READ_SEARCH container_name: harbor-jobservice hostname: harbor-jobservice - image: docker.io/bitnami/harbor-jobservice:2 + image: goharbor/harbor-jobservice:v2.7.4 restart: "no" networks: - harbor depends_on: - - harbor-redis - harbor-core - environment: - - CORE_SECRET=CHANGEME - - JOBSERVICE_SECRET=CHANGEME - - CORE_URL=http://harbor-core:8080 - - REGISTRY_CONTROLLER_URL=http://harbor-registryctl:8080 - - REGISTRY_CREDENTIAL_USERNAME=harbor_registry_user - - REGISTRY_CREDENTIAL_PASSWORD=harbor_registry_password + env_file: + - ./conf/common/env + - ./conf/jobservice/env labels: com.stack.name: "harbor" com.stack.service.name: "jobservice" volumes: - - ./conf/jobservice/config.yml:/etc/jobservice/config.yml:ro - - ./datas/core:/data:rw + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./conf/jobservice/config.yml:/etc/jobservice/config.yml:rw + - ./datas/job_logs:/var/log/jobs:rw deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "jobservice" harbor-redis: extends: - file: ../_vm/common.yml + file: ./_vm/common.yml service: x-common - user: 0:0 + user: 999:999 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - - SETGID - CHOWN - - SYS_ADMIN - - MKNOD - - SYS_CHROOT + - SETGID + - SETUID container_name: harbor-redis hostname: harbor-redis - image: docker.io/bitnami/redis:7.0 + image: goharbor/redis-photon:v2.7.4 restart: "no" + depends_on: + - harbor-log networks: - harbor - environment: - # ALLOW_EMPTY_PASSWORD is recommended only for development. - - ALLOW_EMPTY_PASSWORD=yes - - WORKDIR=/data labels: com.stack.name: "harbor" com.stack.service.name: "redis" volumes: - - ./datas/redis:/data:rw + - /data/redis:/var/lib/redis:rw deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "redis" - harbor-nginx: + harbor-proxy: extends: - file: ../_vm/common.yml + file: ./_vm/common.yml service: x-common user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - - SETGID - CHOWN - - SYS_ADMIN - - MKNOD - - SYS_CHROOT - container_name: harbor-nginx - hostname: harbor-nginx - image: docker.io/bitnami/nginx:1.25 + - SETGID + - SETUID + - NET_BIND_SERVICE + container_name: harbor-proxy + hostname: harbor-proxy + image: goharbor/nginx-photon:v2.7.4 restart: "no" ports: - - '80:8080' + - 80:8080 + - 443:8443 + - 9090:9090 networks: - harbor depends_on: - - harbor-postgresql - harbor-registry - harbor-core - harbor-portal + - harbor-log labels: com.stack.name: "harbor" - com.stack.service.name: "nginx" + com.stack.service.name: "proxy" volumes: - - ./conf/proxy/nginx.conf:/opt/bitnami/nginx/conf/nginx.conf:ro + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./datas/secret/cert:/etc/cert:ro + - ./conf/nginx:/etc/nginx:rw deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "proxy" - harbor-adapter-trivy: + harbor-trivy-adapter: extends: - file: ../_vm/common.yml - service: x-common + file: ./_vm/host.yml + service: x-host user: 0:0 cap_add: - - DAC_OVERRIDE - - FOWNER - - SETUID - SETGID + - SETUID + - DAC_OVERRIDE - CHOWN - SYS_ADMIN - MKNOD - - SYS_CHROOT - container_name: harbor-adapter-trivy - hostname: harbor-adapter-trivy - image: docker.io/bitnami/harbor-adapter-trivy:2 + - DAC_READ_SEARCH + container_name: harbor-trivy-adapter + hostname: harbor-trivy-adapter + image: goharbor/trivy-adapter-photon:v2.7.4 restart: "no" ports: - 8888:8080 networks: - harbor depends_on: + - harbor-log - harbor-redis - environment: - - SCANNER_REDIS_URL=redis://harbor-redis:6379 + env_file: + ./conf/trivy-adapter/env labels: com.stack.name: "harbor" - com.stack.service.name: "adapter-trivy" + com.stack.service.name: "trivy-adapter" volumes: - - ./datas/harbor-adapter-trivy:/bitnami:rw + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + - ./datas/trivy-adapter/trivy:/home/scanner/.cache/trivy:rw + - ./datas/trivy-adapter/reports:/home/scanner/.cache/reports:rw + tmpfs: + - /tmp:rw,size=1G deploy: resources: limits: memory: 1G pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "trivy-adapter" - # harbor-trivy: - # extends: - # file: ../_vm/common.yml - # service: x-common - # user: 0:0 - # cap_add: - # - DAC_OVERRIDE - # - FOWNER - # - SETUID - # - SETGID - # - CHOWN - # - SYS_ADMIN - # - MKNOD - # - SYS_CHROOT - # container_name: harbor-trivy - # hostname: harbor-trivy - # image: bitnami/trivy:0 - # restart: "no" - # networks: - # - harbor - # labels: - # com.stack.name: "harbor" - # com.stack.service.name: "trivy" - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock:rw - # deploy: - # resources: - # limits: - # memory: 1G - # pids: 8192 + harbor-exporter: + extends: + file: ./_vm/host.yml + service: x-host + user: 0:0 + cap_add: + - SETGID + - SETUID + - DAC_OVERRIDE + - CHOWN + - SYS_ADMIN + - MKNOD + - DAC_READ_SEARCH + container_name: harbor-exporter + hostname: harbor-exporter + image: goharbor/harbor-exporter:v2.7.4 + restart: "no" + networks: + - harbor + depends_on: + - harbor-core + - harbor-postgresql + env_file: + - ./conf/common/env + - ./conf/exporter/env + labels: + com.stack.name: "harbor" + com.stack.service.name: "exporter" + volumes: + - ./conf/shared/trust-certificates:/harbor_cust_cert:ro + deploy: + resources: + limits: + memory: 1G + pids: 8192 + logging: + driver: "syslog" + options: + syslog-address: "tcp://0.0.0.0:1514" + tag: "exporter" diff --git a/ssl/build.sh b/ssl/build.sh new file mode 100755 index 0000000..f92f45c --- /dev/null +++ b/ssl/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# 2023-12-13 + +clear + +# Generate KEY +openssl genrsa -des3 -out server.protected.key 2048 + +# Generate CSR +openssl req -new -key server.protected.key -out server.csr + +# Unprotect KEY +openssl rsa -in server.protected.key -out server.key + +# Generate CRT +openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt + +# Place certificates in place +copy ./server.crt ../datas/secret/keys/ +copy ./server.key ../datas/secret/keys/ + +exit 0 \ No newline at end of file