Read and Write Pods in CrashLoopBackOff in v5.9.2-bb.0
Here's the log from the read pod:
kubectl logs -n logging logging-loki-read-646c4785bb-dcfb4
failed services
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:554
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
level=warn ts=2023-08-15T15:48:31.29557334Z caller=loki.go:286 msg="per-tenant timeout not configured, using default engine timeout (\"5m0s\"). This behavior will change in the next major to always use the default per-tenant timeout (\"5m\")."
level=info ts=2023-08-15T15:48:31.297742922Z caller=main.go:108 msg="Starting Loki" version="(version=2.8.3, branch=HEAD, revision=0d81144cf)"
level=info ts=2023-08-15T15:48:31.299145021Z caller=server.go:323 http=[::]:3100 grpc=[::]:9095 msg="server listening on addresses"
level=warn ts=2023-08-15T15:48:31.301771723Z caller=cache.go:114 msg="fifocache config is deprecated. use embedded-cache instead"
level=warn ts=2023-08-15T15:48:31.30179263Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - chunksembedded-cache"
level=info ts=2023-08-15T15:48:31.30228235Z caller=table_manager.go:134 msg="uploading tables"
level=info ts=2023-08-15T15:48:31.302341339Z caller=table_manager.go:262 msg="query readiness setup completed" duration=1.47µs distinct_users_len=0
level=info ts=2023-08-15T15:48:31.302369654Z caller=shipper.go:131 msg="starting index shipper in RW mode"
level=info ts=2023-08-15T15:48:31.302457233Z caller=shipper_index_client.go:78 msg="starting boltdb shipper in RW mode"
level=info ts=2023-08-15T15:48:31.302536372Z caller=table_manager.go:166 msg="handing over indexes to shipper"
level=info ts=2023-08-15T15:48:31.302795501Z caller=table_manager.go:134 msg="uploading tables"
level=info ts=2023-08-15T15:48:31.302875865Z caller=table_manager.go:262 msg="query readiness setup completed" duration=1.287µs distinct_users_len=0
level=info ts=2023-08-15T15:48:31.302892841Z caller=shipper.go:131 msg="starting index shipper in RW mode"
level=info ts=2023-08-15T15:48:31.303059863Z caller=head_manager.go:264 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=info ts=2023-08-15T15:48:31.30308964Z caller=manager.go:80 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2023-08-15T15:48:31.303999568Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/var/loki/rules-temp
level=info ts=2023-08-15T15:48:31.304847344Z caller=worker.go:115 msg="Starting querier worker connected to query-scheduler" scheduler=query-scheduler-discovery.logging.svc.cluster.local.:9095
level=info ts=2023-08-15T15:48:31.308621947Z caller=module_service.go:82 msg=initialising module=runtime-config
level=info ts=2023-08-15T15:48:31.308624595Z caller=module_service.go:82 msg=initialising module=cache-generation-loader
level=info ts=2023-08-15T15:48:31.308688798Z caller=module_service.go:82 msg=initialising module=server
level=info ts=2023-08-15T15:48:31.308759368Z caller=module_service.go:82 msg=initialising module=memberlist-kv
level=error ts=2023-08-15T15:48:31.308791552Z caller=loki.go:512 msg="module failed" module=memberlist-kv error="invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308833083Z caller=loki.go:512 msg="module failed" module=distributor error="failed to start distributor, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.308840829Z caller=loki.go:512 msg="module failed" module=ruler error="failed to start ruler, because it depends on module store, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.308849858Z caller=loki.go:512 msg="module failed" module=usage-report error="failed to start usage-report, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308859615Z caller=loki.go:512 msg="module failed" module=runtime-config error="context canceled"
level=error ts=2023-08-15T15:48:31.3088653Z caller=loki.go:512 msg="module failed" module=query-scheduler error="failed to start query-scheduler, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308873255Z caller=loki.go:512 msg="module failed" module=ring error="failed to start ring, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.308889643Z caller=loki.go:512 msg="module failed" module=index-gateway-ring error="failed to start index-gateway-ring, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308901351Z caller=loki.go:512 msg="module failed" module=querier error="failed to start querier, because it depends on module index-gateway-ring, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.308907127Z caller=loki.go:512 msg="module failed" module=query-frontend error="failed to start query-frontend, because it depends on module usage-report, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.30891228Z caller=loki.go:512 msg="module failed" module=table-manager error="failed to start table-manager, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308920648Z caller=loki.go:512 msg="module failed" module=ingester-querier error="failed to start ingester-querier, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308929308Z caller=loki.go:512 msg="module failed" module=ingester error="failed to start ingester, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=info ts=2023-08-15T15:48:31.308931913Z caller=module_service.go:114 msg="module stopped" module=cache-generation-loader
level=error ts=2023-08-15T15:48:31.30894164Z caller=loki.go:512 msg="module failed" module=compactor error="failed to start compactor, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0005623c0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.144.141 10.134.144.141] 7946 2000000000 5000000000 false 0xc0000d2960 loki false { false }} 0xc0000d2960 loki [{ringDesc 0xee6320} {}]} 0xc00074c050 0xc0000d2960 0xc000847f80 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000018c60 0xc000547500 0xc0005475c0 0xc000547680 0xc000547740 0xc000547980 0xc000547800 0xc000547a40 0xc0005478c0 0xc000873740 0xc000873780 0xc000547bc0 0xc000547c80 0xc000547e00 0xc000547d40 0xc00086c198 0xc0008aebd0 0xc00086c188 0xc00086c190 0xc0008737c0 0xc000873800 10} failed: failed to create transport: failed to start TCP listener on \"10.134.144.141\" port 7946: listen tcp 10.134.144.141:7946: bind: address already in use"
level=error ts=2023-08-15T15:48:31.308949174Z caller=loki.go:512 msg="module failed" module=store error="failed to start store, because it depends on module index-gateway-ring, which has failed: context canceled"
level=error ts=2023-08-15T15:48:31.308954004Z caller=loki.go:512 msg="module failed" module=query-frontend-tripperware error="failed to start query-frontend-tripperware, because it depends on module runtime-config, which has failed: context canceled"
level=info ts=2023-08-15T15:48:31.30912023Z caller=modules.go:1335 msg="server stopped"
level=info ts=2023-08-15T15:48:31.309130279Z caller=module_service.go:114 msg="module stopped" module=server
level=info ts=2023-08-15T15:48:31.309140353Z caller=loki.go:501 msg="Loki stopped"
level=error ts=2023-08-15T15:48:31.309195999Z caller=log.go:171 msg="error running loki" err="failed services\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:554\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598"
Here's the write pod:
kubectl logs -n logging logging-loki-write-0
failed services
github.com/grafana/loki/pkg/loki.(*Loki).Run
/src/loki/pkg/loki/loki.go:554
main.main
/src/loki/cmd/loki/main.go:110
runtime.main
/usr/local/go/src/runtime/proc.go:250
runtime.goexit
/usr/local/go/src/runtime/asm_amd64.s:1598
level=warn ts=2023-08-15T15:49:20.406155241Z caller=loki.go:286 msg="per-tenant timeout not configured, using default engine timeout (\"5m0s\"). This behavior will change in the next major to always use the default per-tenant timeout (\"5m\")."
level=info ts=2023-08-15T15:49:20.407882624Z caller=main.go:108 msg="Starting Loki" version="(version=2.8.3, branch=HEAD, revision=0d81144cf)"
level=info ts=2023-08-15T15:49:20.409215062Z caller=server.go:323 http=[::]:3100 grpc=[::]:9095 msg="server listening on addresses"
level=warn ts=2023-08-15T15:49:20.411324664Z caller=cache.go:114 msg="fifocache config is deprecated. use embedded-cache instead"
level=warn ts=2023-08-15T15:49:20.411352132Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - chunksembedded-cache"
level=info ts=2023-08-15T15:49:20.411914018Z caller=table_manager.go:134 msg="uploading tables"
level=info ts=2023-08-15T15:49:20.411970525Z caller=table_manager.go:262 msg="query readiness setup completed" duration=2.423µs distinct_users_len=0
level=info ts=2023-08-15T15:49:20.412000696Z caller=shipper.go:131 msg="starting index shipper in RW mode"
level=info ts=2023-08-15T15:49:20.412173113Z caller=shipper_index_client.go:78 msg="starting boltdb shipper in RW mode"
level=info ts=2023-08-15T15:49:20.412248916Z caller=table_manager.go:166 msg="handing over indexes to shipper"
level=info ts=2023-08-15T15:49:20.412577554Z caller=table_manager.go:262 msg="query readiness setup completed" duration=1.604µs distinct_users_len=0
level=info ts=2023-08-15T15:49:20.412593921Z caller=shipper.go:131 msg="starting index shipper in RW mode"
level=info ts=2023-08-15T15:49:20.412604999Z caller=table_manager.go:134 msg="uploading tables"
level=info ts=2023-08-15T15:49:20.41284316Z caller=head_manager.go:264 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=info ts=2023-08-15T15:49:20.413458035Z caller=manager.go:80 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=2 indices=3 failures=0
level=info ts=2023-08-15T15:49:20.414446207Z caller=worker.go:115 msg="Starting querier worker connected to query-scheduler" scheduler=query-scheduler-discovery.logging.svc.cluster.local.:9095
level=info ts=2023-08-15T15:49:20.417383191Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/var/loki/rules-temp
level=info ts=2023-08-15T15:49:20.419966334Z caller=module_service.go:82 msg=initialising module=runtime-config
level=info ts=2023-08-15T15:49:20.42003546Z caller=module_service.go:82 msg=initialising module=cache-generation-loader
level=info ts=2023-08-15T15:49:20.420068314Z caller=module_service.go:82 msg=initialising module=server
level=info ts=2023-08-15T15:49:20.420141607Z caller=module_service.go:82 msg=initialising module=memberlist-kv
level=error ts=2023-08-15T15:49:20.42019302Z caller=loki.go:512 msg="module failed" module=memberlist-kv error="invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420222783Z caller=loki.go:512 msg="module failed" module=query-frontend error="failed to start query-frontend, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.420237181Z caller=loki.go:512 msg="module failed" module=table-manager error="failed to start table-manager, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.420248999Z caller=loki.go:512 msg="module failed" module=runtime-config error="context canceled"
level=error ts=2023-08-15T15:49:20.420258504Z caller=loki.go:512 msg="module failed" module=querier error="failed to start querier, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420283223Z caller=loki.go:512 msg="module failed" module=ruler error="failed to start ruler, because it depends on module ingester-querier, which has failed: context canceled"
level=info ts=2023-08-15T15:49:20.420308678Z caller=module_service.go:114 msg="module stopped" module=cache-generation-loader
level=error ts=2023-08-15T15:49:20.42030549Z caller=loki.go:512 msg="module failed" module=usage-report error="failed to start usage-report, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420327976Z caller=loki.go:512 msg="module failed" module=store error="failed to start store, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.420336434Z caller=loki.go:512 msg="module failed" module=query-scheduler error="failed to start query-scheduler, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420348582Z caller=loki.go:512 msg="module failed" module=ring error="failed to start ring, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420357125Z caller=loki.go:512 msg="module failed" module=query-frontend-tripperware error="failed to start query-frontend-tripperware, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.42036376Z caller=loki.go:512 msg="module failed" module=ingester error="failed to start ingester, because it depends on module usage-report, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.420368655Z caller=loki.go:512 msg="module failed" module=distributor error="failed to start distributor, because it depends on module runtime-config, which has failed: context canceled"
level=error ts=2023-08-15T15:49:20.420373629Z caller=loki.go:512 msg="module failed" module=compactor error="failed to start compactor, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420380147Z caller=loki.go:512 msg="module failed" module=ingester-querier error="failed to start ingester-querier, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=error ts=2023-08-15T15:49:20.420388235Z caller=loki.go:512 msg="module failed" module=index-gateway-ring error="failed to start index-gateway-ring, because it depends on module memberlist-kv, which has failed: invalid service state: Failed, expected: Running, failure: invalid service state: Failed, expected: Running, failure: service &{0xc0007fdea0 { true 10000000000 4 30000000000 200000000 3 30000000000 0 true 7946 false [logging-loki-memberlist] 1000000000 60000000000 10 false 0 300000000000 20000000000 0 {[10.134.159.71 10.134.159.71] 7946 2000000000 5000000000 false 0xc000050a50 loki false { false }} 0xc000050a50 loki [{ringDesc 0xee6320} {}]} 0xc0002865a0 0xc000050a50 0xc0005e43f0 {[] {[] 0}} <nil> <nil> {0 0} map[] map[ringDesc:{ringDesc 0xee6320} usagestats.jsonCodec:{}] {0 0} map[] map[] {0 0} [] 0 [] 0 0 {0 0} map[] 0xc000715560 0xc000564ba0 0xc000564c60 0xc000564d20 0xc000564de0 0xc000565020 0xc000564ea0 0xc0005650e0 0xc000564f60 0xc0001c0e80 0xc0001c0ec0 0xc000565260 0xc000565320 0xc0005654a0 0xc0005653e0 0xc0000c45a0 0xc0005e51f0 0xc0000c4590 0xc0000c4598 0xc0001c0f40 0xc0001c0f80 10} failed: failed to create transport: failed to start TCP listener on \"10.134.159.71\" port 7946: listen tcp 10.134.159.71:7946: bind: address already in use"
level=info ts=2023-08-15T15:49:20.420508267Z caller=modules.go:1335 msg="server stopped"
level=info ts=2023-08-15T15:49:20.420517906Z caller=module_service.go:114 msg="module stopped" module=server
level=info ts=2023-08-15T15:49:20.420525009Z caller=loki.go:501 msg="Loki stopped"
level=error ts=2023-08-15T15:49:20.420571036Z caller=log.go:171 msg="error running loki" err="failed services\ngithub.com/grafana/loki/pkg/loki.(*Loki).Run\n\t/src/loki/pkg/loki/loki.go:554\nmain.main\n\t/src/loki/cmd/loki/main.go:110\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:250\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1598"