==> /var/log/gitlab/gitlab-rails/exceptions_json.log <====> /var/log/gitlab/gitlab-rails/application.log <==2025-08-28T04:45:08.052Z: {:message=>"Dropping detached postgres partitions"}2025-08-28T04:45:08.053Z: {:message=>"Switched database connection", :connection_name=>"main"}2025-08-28T04:45:08.053Z: {:message=>"Checking for previously detached partitions to drop"}2025-08-28T04:45:08.144Z: {:message=>"Finished dropping detached postgres partitions"}2025-08-28T04:45:08.144Z: {:message=>"Switched database connection", :connection_name=>"main"}2025-08-28T04:45:08.237Z: {:message=>"Switched database connection", :connection_name=>"main"}2025-08-28T04:45:08.266Z: {:message=>"Switched database connection", :connection_name=>"main"}2025-08-28T04:45:08.386Z: {:message=>"Switched database connection", :connection_name=>"main"}2025-08-28T05:20:37.702Z: {:message=>"Excluding unhealthy shards", :failed_checks=>[{:status=>"failed", :message=>"7:permission denied. debug_error_string:{\"created\":\"@1756358407.690139809\",\"description\":\"Error received from peer unix:/var/opt/gitlab/gitaly/gitaly.socket\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1063,\"grpc_message\":\"permission denied\",\"grpc_status\":7}", :labels=>{:shard=>"gitaly-2"}}, {:status=>"failed", :message=>"gitaly node connectivity & disk access: the following nodes are not healthy: tcp://192.168.41.3:8075", :labels=>{:shard=>"default"}}], :class=>"RepositoryCheck::DispatchWorker"}2025-08-28T05:40:08.600Z: Ci::StuckBuilds::DropScheduledService: Cleaning scheduled, timed-out builds==> /var/log/gitlab/gitlab-rails/grpc.log <====> /var/log/gitlab/gitlab-rails/git_json.log <====> /var/log/gitlab/gitlab-rails/auth.log <====> /var/log/gitlab/gitlab-rails/api_json.log <====> /var/log/gitlab/gitlab-rails/service_measurement.log <====> /var/log/gitlab/gitlab-rails/database_load_balancing.log <====> /var/log/gitlab/gitlab-rails/production_json.log <=={"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:09:57.670Z","params":[],"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.098239,"mem_objects":1992,"mem_bytes":867576,"mem_mallocs":4906,"mem_total_bytes":947256,"pid":29061,"worker_id":"puma_2","rate_limiting_gates":[],"correlation_id":"7710a24b-ad5e-45c2-b3fb-731d8e630232","db_duration_s":0.0,"view_duration_s":0.00095,"duration_s":0.08899}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:10:12.646Z","params":[],"redis_calls":3,"redis_duration_s":0.00208,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.00208,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.07088,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29064,"worker_id":"puma_3","rate_limiting_gates":[],"correlation_id":"acab1e51-5825-47d1-96dc-1faa70268d7a","db_duration_s":0.0,"view_duration_s":0.00102,"duration_s":0.05915}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:10:27.640Z","params":[],"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.066442,"mem_objects":1992,"mem_bytes":867576,"mem_mallocs":4906,"mem_total_bytes":947256,"pid":29064,"worker_id":"puma_3","rate_limiting_gates":[],"correlation_id":"8151f690-67a9-478a-a6cb-19d703acc7c5","db_duration_s":0.0,"view_duration_s":0.00098,"duration_s":0.05607}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:10:42.640Z","params":[],"redis_calls":3,"redis_duration_s":0.001468,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.001468,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.066602,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29053,"worker_id":"puma_0","rate_limiting_gates":[],"correlation_id":"e0a6709c-3d88-4d2c-9b5f-c6a220ecb137","db_duration_s":0.0,"view_duration_s":0.00093,"duration_s":0.05519}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:10:57.633Z","params":[],"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.059788,"mem_objects":1992,"mem_bytes":867576,"mem_mallocs":4906,"mem_total_bytes":947256,"pid":29053,"worker_id":"puma_0","rate_limiting_gates":[],"correlation_id":"5a3bfca8-2182-4000-9ede-11d89bbf155c","db_duration_s":0.0,"view_duration_s":0.00085,"duration_s":0.05267}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:11:12.642Z","params":[],"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.068716,"mem_objects":1992,"mem_bytes":867576,"mem_mallocs":4906,"mem_total_bytes":947256,"pid":29053,"worker_id":"puma_0","rate_limiting_gates":[],"correlation_id":"ac47d522-332d-4b1d-9d3a-d855ff239712","db_duration_s":0.0,"view_duration_s":0.00093,"duration_s":0.05931}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:11:27.659Z","params":[],"redis_calls":3,"redis_duration_s":0.002131,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.002131,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.083974,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29064,"worker_id":"puma_3","rate_limiting_gates":[],"correlation_id":"2667866f-fcbe-4817-ab97-7df209d37897","db_duration_s":0.0,"view_duration_s":0.00146,"duration_s":0.06867}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:11:42.648Z","params":[],"redis_calls":3,"redis_duration_s":0.001244,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.001244,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.075447,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29057,"worker_id":"puma_1","rate_limiting_gates":[],"correlation_id":"1f51f1b2-c732-4cee-927f-480ae2563dd4","db_duration_s":0.0,"view_duration_s":0.00118,"duration_s":0.06432}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:11:57.647Z","params":[],"redis_calls":3,"redis_duration_s":0.001642,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.001642,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.07402,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29061,"worker_id":"puma_2","rate_limiting_gates":[],"correlation_id":"91a88641-1e5f-4c61-b7f6-b58c9ba15e68","db_duration_s":0.0,"view_duration_s":0.00082,"duration_s":0.0627}{"method":"GET","path":"/-/metrics","format":"html","controller":"MetricsController","action":"index","status":200,"time":"2025-08-28T06:12:12.650Z","params":[],"redis_calls":3,"redis_duration_s":0.00147,"redis_read_bytes":608,"redis_write_bytes":206,"redis_cache_calls":3,"redis_cache_duration_s":0.00147,"redis_cache_read_bytes":608,"redis_cache_write_bytes":206,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.076819,"mem_objects":2529,"mem_bytes":1023080,"mem_mallocs":5170,"mem_total_bytes":1124240,"pid":29053,"worker_id":"puma_0","rate_limiting_gates":[],"correlation_id":"90764384-aa97-40bb-9d13-f381f8285583","db_duration_s":0.0,"view_duration_s":0.0012,"duration_s":0.06532}==> /var/log/gitlab/gitlab-rails/sidekiq_client.log <====> /var/log/gitlab/gitlab-rails/gitlab-rails-db-migrate-2025-08-09-16-03-52.log <====> /var/log/gitlab/gitlab-rails/application_json.log <=={"severity":"INFO","time":"2025-08-28T04:45:08.052Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Dropping detached postgres partitions"}{"severity":"DEBUG","time":"2025-08-28T04:45:08.053Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Switched database connection","connection_name":"main"}{"severity":"INFO","time":"2025-08-28T04:45:08.053Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Checking for previously detached partitions to drop"}{"severity":"INFO","time":"2025-08-28T04:45:08.144Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Finished dropping detached postgres partitions"}{"severity":"DEBUG","time":"2025-08-28T04:45:08.144Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Switched database connection","connection_name":"main"}{"severity":"DEBUG","time":"2025-08-28T04:45:08.238Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Switched database connection","connection_name":"main"}{"severity":"DEBUG","time":"2025-08-28T04:45:08.266Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Switched database connection","connection_name":"main"}{"severity":"DEBUG","time":"2025-08-28T04:45:08.387Z","correlation_id":"f49f6fc97aa56ff192e961d548cef24d","message":"Switched database connection","connection_name":"main"}{"severity":"ERROR","time":"2025-08-28T05:20:37.702Z","correlation_id":"5e2caba4a96c082232b68c79ad5d9c34","message":"Excluding unhealthy shards","failed_checks":[{"status":"failed","message":"7:permission denied. debug_error_string:{\"created\":\"@1756358407.690139809\",\"description\":\"Error received from peer unix:/var/opt/gitlab/gitaly/gitaly.socket\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1063,\"grpc_message\":\"permission denied\",\"grpc_status\":7}","labels":{"shard":"gitaly-2"}},{"status":"failed","message":"gitaly node connectivity \u0026 disk access: the following nodes are not healthy: tcp://192.168.41.3:8075","labels":{"shard":"default"}}],"class":"RepositoryCheck::DispatchWorker"}{"severity":"INFO","time":"2025-08-28T05:40:08.600Z","correlation_id":"996ed303f842c5f001a402c9e726bdbb","message":"Ci::StuckBuilds::DropScheduledService: Cleaning scheduled, timed-out builds"}==> /var/log/gitlab/gitlab-rails/production.log <==Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:15 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:16 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:17 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:18 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:19 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:20 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:21 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:22 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:23 +0800Started GET "/-/health" for 127.0.0.1 at 2025-08-28 14:12:24 +0800==> /var/log/gitlab/prometheus/current <==2025-08-28_03:57:30.77464 ts=2025-08-28T03:57:30.774Z caller=checkpoint.go:100 level=info component=tsdb msg="Creating checkpoint" from_segment=214 to_segment=215 mint=17562168000002025-08-28_03:57:30.93603 ts=2025-08-28T03:57:30.935Z caller=head.go:1013 level=info component=tsdb msg="WAL checkpoint complete" first=214 last=215 duration=161.418949ms2025-08-28_03:57:32.87013 ts=2025-08-28T03:57:32.870Z caller=manager.go:213 level=error component="scrape manager" msg="error creating new scrape pool" err="error creating HTTP client: unable to load specified CA cert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: open /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: no such file or directory" scrape_pool=kubernetes-cadvisor2025-08-28_03:57:32.87017 ts=2025-08-28T03:57:32.870Z caller=manager.go:213 level=error component="scrape manager" msg="error creating new scrape pool" err="error creating HTTP client: unable to load specified CA cert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: open /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: no such file or directory" scrape_pool=kubernetes-nodes2025-08-28_03:57:32.87040 ts=2025-08-28T03:57:32.870Z caller=manager.go:213 level=error component="scrape manager" msg="error creating new scrape pool" err="error creating HTTP client: unable to load specified CA cert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: open /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: no such file or directory" scrape_pool=kubernetes-pods2025-08-28_05:57:32.13015 ts=2025-08-28T05:57:32.129Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1756349850661 maxt=1756353600000 ulid=01K3QM1CY67XX3HNZ43AT89XEM duration=411.731549ms2025-08-28_05:57:32.14118 ts=2025-08-28T05:57:32.141Z caller=db.go:1294 level=info component=tsdb msg="Deleting obsolete block" block=01K2D169FEGBGWT15E3SG4EECN2025-08-28_05:57:32.14605 ts=2025-08-28T05:57:32.145Z caller=db.go:1294 level=info component=tsdb msg="Deleting obsolete block" block=01K2GWSC54DD7BBKBVNC8NSPM02025-08-28_05:57:32.15102 ts=2025-08-28T05:57:32.150Z caller=db.go:1294 level=info component=tsdb msg="Deleting obsolete block" block=01K2EYZW7RMFE1D0FD6YTRFB3N2025-08-28_05:57:32.18483 ts=2025-08-28T05:57:32.184Z caller=head.go:844 level=info component=tsdb msg="Head GC completed" duration=33.616389ms==> /var/log/gitlab/prometheus/state <====> /var/log/gitlab/puma/puma_stderr.log <====> /var/log/gitlab/puma/puma_stdout.log <====> /var/log/gitlab/puma/current <==2025-08-28_03:57:29.57607 {"timestamp":"2025-08-28T03:57:29.576Z","pid":25248,"message":"* Environment: production"}2025-08-28_03:57:29.57610 {"timestamp":"2025-08-28T03:57:29.576Z","pid":25248,"message":"* Master PID: 25248"}2025-08-28_03:57:29.57611 {"timestamp":"2025-08-28T03:57:29.576Z","pid":25248,"message":"* Workers: 4"}2025-08-28_03:57:29.57616 {"timestamp":"2025-08-28T03:57:29.576Z","pid":25248,"message":"* Restarts: (鉁�) hot (鉁�) phased"}2025-08-28_03:57:29.57616 {"timestamp":"2025-08-28T03:57:29.576Z","pid":25248,"message":"* Preloading application"}2025-08-28_03:58:43.04424 {"timestamp":"2025-08-28T03:58:43.043Z","pid":25248,"message":"* Listening on unix:///var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"}2025-08-28_03:58:43.04504 {"timestamp":"2025-08-28T03:58:43.044Z","pid":25248,"message":"* Listening on http://127.0.0.1:8080"}2025-08-28_03:58:43.04515 {"timestamp":"2025-08-28T03:58:43.045Z","pid":25248,"message":"! WARNING: Detected 1 Thread(s) started in app boot:"}2025-08-28_03:58:43.04528 {"timestamp":"2025-08-28T03:58:43.045Z","pid":25248,"message":"! #\u003cThread:0x00007fe68f0ea588 /opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.6.3/lib/rack/timeout/support/scheduler.rb:73 sleep\u003e - /opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.6.3/lib/rack/timeout/support/scheduler.rb:91:in `sleep'"}2025-08-28_03:58:43.04543 {"timestamp":"2025-08-28T03:58:43.045Z","pid":25248,"message":"Use Ctrl-C to stop"}==> /var/log/gitlab/puma/state <====> /var/log/gitlab/gitlab-kas/current <==2025-08-28_03:54:24.78884 {"level":"info","time":"2025-08-28T11:54:24.788+0800","msg":"Private API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8155"}2025-08-28_03:54:24.78891 {"level":"info","time":"2025-08-28T11:54:24.788+0800","msg":"API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8153"}2025-08-28_03:54:24.78892 {"level":"info","time":"2025-08-28T11:54:24.788+0800","msg":"Agentk API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8150","is_websocket":true}2025-08-28_03:54:24.78893 {"level":"info","time":"2025-08-28T11:54:24.788+0800","msg":"Observability endpoint is up","mod_name":"observability","net_network":"tcp","net_address":"127.0.0.1:8151"}2025-08-28_03:54:24.78893 {"level":"info","time":"2025-08-28T11:54:24.788+0800","msg":"Kubernetes API endpoint is up","mod_name":"kubernetes_api","net_network":"tcp","net_address":"127.0.0.1:8154"}2025-08-28_03:57:24.87723 {"level":"info","time":"2025-08-28T11:57:24.877+0800","msg":"Private API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8155"}2025-08-28_03:57:24.87727 {"level":"info","time":"2025-08-28T11:57:24.877+0800","msg":"Kubernetes API endpoint is up","mod_name":"kubernetes_api","net_network":"tcp","net_address":"127.0.0.1:8154"}2025-08-28_03:57:24.87728 {"level":"info","time":"2025-08-28T11:57:24.877+0800","msg":"API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8153"}2025-08-28_03:57:24.87728 {"level":"info","time":"2025-08-28T11:57:24.877+0800","msg":"Agentk API endpoint is up","net_network":"tcp","net_address":"127.0.0.1:8150","is_websocket":true}2025-08-28_03:57:24.87744 {"level":"info","time":"2025-08-28T11:57:24.877+0800","msg":"Observability endpoint is up","mod_name":"observability","net_network":"tcp","net_address":"127.0.0.1:8151"}==> /var/log/gitlab/gitlab-kas/state <====> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-28-11-56-22.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-09-16-03-47.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-25-16-02-47.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-09-19-26-13.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/current <=={"component":"HealthManager","correlation_id":"01K3QMW441YR5FPC7281XC8428","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:07.425Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMW60ZRCRGG5A8CCWG7KFS","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:09.375Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMW7V9SG05V7RTV1FWGZC5","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:11.241Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMW9KMRXV0PBDZK1H7AH5J","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:13.045Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWBHRFXSF6F2N002A5AT3","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:15.033Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWDDMN13NJGVYHMWFZXD4","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:16.948Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWF6JZP9K1Z9W0TV43XH1","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:18.770Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWH0M10MF7FK1AVW6R663","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:20.628Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWJYMJZBZ9WQD24YZRWM2","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:22.613Z","virtual_storage":"default"}{"component":"HealthManager","correlation_id":"01K3QMWMY57263MBG9SCS66EZY","error":"rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial tcp 192.168.41.3:8075: connect: connection refused\"","level":"error","msg":"failed checking node health","pid":25138,"storage":"gitaly-3","time":"2025-08-28T06:12:24.646Z","virtual_storage":"default"}==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-26-12-08-33.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-09-22-20-50.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/state <====> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-09-21-38-32.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/praefect/praefect-sql-migrate-2025-08-25-19-09-07.log <==praefect sql-migrate: all migrations are up==> /var/log/gitlab/sidekiq/current <=={"severity":"INFO","time":"2025-08-28T06:11:05.493Z","retry":0,"queue":"cronjob:schedule_merge_request_cleanup_refs","version":0,"queue_namespace":"cronjob","args":[],"class":"ScheduleMergeRequestCleanupRefsWorker","jid":"45ab0f892d791776de9a0b42","created_at":"2025-08-28T06:11:05.467Z","meta.caller_id":"Cronjob","correlation_id":"e6f219114252350a4ccfc656a401e2cc","meta.root_caller_id":"Cronjob","meta.feature_category":"code_review","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:cronjob:schedule_merge_request_cleanup_refs:33e8a9dcd4c9780ad0ea123ad7ccbabde1aa1e90ffcbb928434ba4b5800a5811","size_limiter":"validated","enqueued_at":"2025-08-28T06:11:05.469Z","job_size_bytes":2,"pid":25493,"message":"ScheduleMergeRequestCleanupRefsWorker JID-45ab0f892d791776de9a0b42: done: 0.016838 sec","job_status":"done","scheduling_latency_s":0.007451,"redis_calls":3,"redis_duration_s":0.002085,"redis_read_bytes":204,"redis_write_bytes":283,"redis_cache_calls":1,"redis_cache_duration_s":0.000723,"redis_cache_read_bytes":202,"redis_cache_write_bytes":61,"redis_queues_calls":2,"redis_queues_duration_s":0.001362,"redis_queues_read_bytes":2,"redis_queues_write_bytes":222,"db_count":0,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":0,"db_main_count":0,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.0,"db_main_duration_s":0.0,"db_main_replica_duration_s":0.0,"cpu_s":0.010796,"mem_objects":1471,"mem_bytes":122488,"mem_mallocs":380,"mem_total_bytes":181328,"worker_id":"sidekiq_0","rate_limiting_gates":[],"duration_s":0.016838,"completed_at":"2025-08-28T06:11:05.493Z","load_balancing_strategy":"primary","db_duration_s":0.0}{"severity":"INFO","time":"2025-08-28T06:12:04.467Z","retry":0,"queue":"cronjob:users_migrate_records_to_ghost_user_in_batches","version":0,"queue_namespace":"cronjob","args":[],"class":"Users::MigrateRecordsToGhostUserInBatchesWorker","jid":"bdba58df550f25a5c4502ce3","created_at":"2025-08-28T06:12:04.425Z","meta.caller_id":"Cronjob","correlation_id":"f3325d7eef2e720bda14b20f84b55d23","meta.root_caller_id":"Cronjob","meta.feature_category":"users","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:cronjob:users_migrate_records_to_ghost_user_in_batches:4bdb3193c92ce2ad4e73652ad55816e507f3dd7f07576150fc7c08572353b9e8","size_limiter":"validated","enqueued_at":"2025-08-28T06:12:04.462Z","job_size_bytes":2,"pid":25493,"message":"Users::MigrateRecordsToGhostUserInBatchesWorker JID-bdba58df550f25a5c4502ce3: start","job_status":"start","scheduling_latency_s":0.005215}{"severity":"INFO","time":"2025-08-28T06:12:04.631Z","retry":0,"queue":"cronjob:database_batched_background_migration_ci_database","version":0,"queue_namespace":"cronjob","args":[],"class":"Database::BatchedBackgroundMigration::CiDatabaseWorker","jid":"b8e2653e2bbe232da2228b57","created_at":"2025-08-28T06:12:04.622Z","meta.caller_id":"Cronjob","correlation_id":"f89e8db756fe29aef54362387433b49c","meta.root_caller_id":"Cronjob","meta.feature_category":"database","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:cronjob:database_batched_background_migration_ci_database:6ba8adee4a8c1e77d2f087a2765c43226ceffa1fd65abc34b95725a7c9abd857","enqueued_at":"2025-08-28T06:12:04.627Z","job_size_bytes":2,"pid":25493,"message":"Database::BatchedBackgroundMigration::CiDatabaseWorker JID-b8e2653e2bbe232da2228b57: start","job_status":"start","scheduling_latency_s":0.004391}{"severity":"INFO","time":"2025-08-28T06:12:04.655Z","class":"Database::BatchedBackgroundMigration::CiDatabaseWorker","database":"ci","message":"skipping migration execution for unconfigured database","retry":0}{"severity":"INFO","time":"2025-08-28T06:12:04.664Z","retry":0,"queue":"cronjob:database_batched_background_migration_ci_database","version":0,"queue_namespace":"cronjob","args":[],"class":"Database::BatchedBackgroundMigration::CiDatabaseWorker","jid":"b8e2653e2bbe232da2228b57","created_at":"2025-08-28T06:12:04.622Z","meta.caller_id":"Cronjob","correlation_id":"f89e8db756fe29aef54362387433b49c","meta.root_caller_id":"Cronjob","meta.feature_category":"database","worker_data_consistency":"always","idempotency_key":"resque:gitlab:duplicate:cronjob:database_batched_background_migration_ci_database:6ba8adee4a8c1e77d2f087a2765c43226ceffa1fd65abc34b95725a7c9abd857","enqueued_at":"2025-08-28T06:12:04.627Z","job_size_bytes":2,"pid":25493,"message":"Database::BatchedBackgroundMigration::CiDatabaseWorker JID-b8e2653e2bbe232da2228b57: done: 0.032335 sec","job_status":"done","scheduling_latency_s":0.004391,"redis_calls":2,"redis_duration_s":0.006605,"redis_read_bytes":2,"redis_write_bytes":236,"redis_queues_calls":2,"redis_queues_duration_s":0.006605,"redis_queues_read_bytes
最新发布
本文介绍了一种常见的数据库连接错误DB_SEC_E_AUTH_FAILED，分析了错误产生的原因，并给出了解决方案。错误源于数据库连接字符串配置不当，导致Connection为空值，进而引发访问数据库失败。
扫一扫
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
