- Notifications
You must be signed in to change notification settings - Fork928
Closed
Description
Seen here:https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414
=== Failed=== FAIL: enterprise/tailnet TestPGCoordinator_MultiAgent (2.27s) t.go:85: 2023-08-29 22:55:56.395 [debu] coord1.pgcoord: cleaned up old coordinators coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.406 [debu] coord1.pgcoord: sent heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.412 [debu] coord1.pgcoord.querier: subscribed to client updates coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.415 [debu] coord1.pgcoord: ignoring our own heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.415 [debu] coord1.pgcoord.querier: subscribed to agent updates coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.415 [info] coord1.pgcoord: starting coordinator coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.415 [debu] coord2.pgcoord.querier: subscribed to client updates coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.415 [debu] coord2.pgcoord.querier: subscribed to agent updates coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.415 [info] coord2.pgcoord: starting coordinator coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.418 [debu] coord3.pgcoord.querier: subscribed to client updates coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f[292](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:293)5a41 t.go:85: 2023-08-29 22:55:56.421 [debu] coord3.pgcoord.querier: subscribed to agent updates coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.421 [info] coord3.pgcoord: starting coordinator coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.421 [info] coord3.pgcoord: serving connection coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 t.go:85: 2023-08-29 22:55:56.421 [info] coord1.pgcoord: serving connection coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.421 [debu] coord3.pgcoord.querier: skipping 0 length node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=false t.go:85: 2023-08-29 22:55:56.421 [info] coord2.pgcoord: serving connection coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.421 [debu] coord1.pgcoord.querier: skipping 0 length node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.421 [debu] coord3.pgcoord: got node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 node={"id":0,"as_of":"0001-01-01T00:00:00Z","key":"nodekey:0000000000000000000000000000000000000000000000000000000000000000","disco":"discokey:0000000000000000000000000000000000000000000000000000000000000000","preferred_derp":3,"derp_latency":null,"derp_forced_websockets":null,"addresses":null,"allowed_ips":null,"endpoints":null} t.go:85: 2023-08-29 22:55:56.421 [debu] coord2.pgcoord.querier: skipping 0 length node update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.429 [debu] coord2.pgcoord: cleaned up old coordinators coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [debu] coord3.pgcoord: got heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [info] coord3.pgcoord: heartbeats (re)started coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [debu] coord3.pgcoord: set initial heartbeat timeout coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.435 [debu] coord2.pgcoord: sent heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [debu] coord1.pgcoord: got heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [info] coord1.pgcoord: heartbeats (re)started coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.435 [debu] coord1.pgcoord: set initial heartbeat timeout coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:56.435 [debu] coord3.pgcoord.querier: skipping 0 length node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=false t.go:85: 2023-08-29 22:55:56.435 [debu] coord1.pgcoord.querier: skipping 0 length node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.435 [debu] coord2.pgcoord: ignoring our own heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.492 [debu] coord3.pgcoord: cleaned up old coordinators coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.500 [debu] coord3.pgcoord.querier: queried agents coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_agents=0 error=<nil> t.go:85: 2023-08-29 22:55:56.500 [debu] coord3.pgcoord.querier: skipping 0 length node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=false t.go:85: 2023-08-29 22:55:56.506 [debu] coord1.pgcoord.querier: queried clients of agent coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_clients=0 error=<nil> t.go:85: 2023-08-29 22:55:56.506 [debu] coord1.pgcoord.querier: skipping 0 length node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.520 [debu] coord2.pgcoord.querier: queried clients of agent coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_clients=0 error=<nil> t.go:85: 2023-08-29 22:55:56.520 [debu] coord2.pgcoord.querier: skipping 0 length node update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.527 [debu] coord3.pgcoord: ignoring our own heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.527 [debu] coord1.pgcoord: got heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 other_coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.527 [info] coord1.pgcoord: heartbeats (re)started coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 other_coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.527 [debu] coord1.pgcoord: computed oldest heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 oldest="2023-08-29T22:55:56.435327504Z" time_to_expiry=5.907814648s t.go:85: 2023-08-29 22:55:56.527 [debu] coord1.pgcoord.querier: skipping 0 length node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.527 [debu] coord2.pgcoord: got heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa other_coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.527 [info] coord2.pgcoord: heartbeats (re)started coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa other_coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.527 [debu] coord2.pgcoord: set initial heartbeat timeout coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:56.527 [debu] coord2.pgcoord.querier: skipping 0 length node update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=true t.go:85: 2023-08-29 22:55:56.527 [debu] coord3.pgcoord: sent heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:56.541 [debu] coord3.pgcoord: upserted client binding coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 node="{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":3,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}" error=<nil> t.go:85: 2023-08-29 22:55:56.541 [debu] coord3.pgcoord.querier: got client update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.541 [debu] coord3.pgcoord.querier: ignoring update because we have no mapper coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.541 [debu] coord1.pgcoord.querier: got client update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.541 [debu] coord2.pgcoord.querier: got client update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.544 [debu] coord2.pgcoord.querier: queried clients of agent coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_clients=1 error=<nil> t.go:85: 2023-08-29 22:55:56.544 [debu] coord1.pgcoord.querier: queried clients of agent coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_clients=1 error=<nil> t.go:85: 2023-08-29 22:55:56.544 [debu] coord2.pgcoord: wrote nodes coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":3,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:56.544 [debu] coord1.pgcoord: got node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node={"id":0,"as_of":"0001-01-01T00:00:00Z","key":"nodekey:0000000000000000000000000000000000000000000000000000000000000000","disco":"discokey:0000000000000000000000000000000000000000000000000000000000000000","preferred_derp":1,"derp_latency":null,"derp_forced_websockets":null,"addresses":null,"allowed_ips":null,"endpoints":null} t.go:85: 2023-08-29 22:55:56.544 [debu] coord1.pgcoord: wrote nodes coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":3,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:56.564 [debu] coord3.pgcoord.querier: got agent update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.564 [debu] coord1.pgcoord: upserted agent binding coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node="{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":1,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}" error=<nil> t.go:85: 2023-08-29 22:55:56.564 [debu] coord1.pgcoord.querier: got agent update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.564 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.564 [debu] coord2.pgcoord.querier: got agent update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.564 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:56.570 [debu] coord3.pgcoord.querier: queried agents coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_agents=1 error=<nil> t.go:85: 2023-08-29 22:55:56.570 [debu] coord3.pgcoord.querier: skipping 0 length node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 clients_of_agent=false t.go:85: 2023-08-29 22:55:58.413 [debu] coord1.pgcoord: sent heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [debu] coord3.pgcoord: got heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 other_coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [info] coord3.pgcoord: heartbeats (re)started coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 other_coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [debu] coord3.pgcoord: computed oldest heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 oldest="2023-08-29T22:55:56.435251171Z" time_to_expiry=4.021889364s t.go:85: 2023-08-29 22:55:58.413 [debu] coord2.pgcoord: got node update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node={"id":0,"as_of":"0001-01-01T00:00:00Z","key":"nodekey:0000000000000000000000000000000000000000000000000000000000000000","disco":"discokey:0000000000000000000000000000000000000000000000000000000000000000","preferred_derp":2,"derp_latency":null,"derp_forced_websockets":null,"addresses":null,"allowed_ips":null,"endpoints":null} t.go:85: 2023-08-29 22:55:58.413 [debu] coord1.pgcoord: ignoring our own heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [debu] coord2.pgcoord: got heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa other_coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [info] coord2.pgcoord: heartbeats (re)started coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa other_coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:85: 2023-08-29 22:55:58.413 [debu] coord2.pgcoord: computed oldest heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa oldest="2023-08-29T22:55:56.527671624Z" time_to_expiry=4.114082431s t.go:85: 2023-08-29 22:55:58.413 [debu] coord2.pgcoord: skipping duplicate update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":3,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.413 [debu] coord3.pgcoord: wrote nodes coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":1,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.427 [debu] coord2.pgcoord: upserted agent binding coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node="{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":2,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}" error=<nil> t.go:85: 2023-08-29 22:55:58.427 [debu] coord3.pgcoord.querier: got agent update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.427 [debu] coord2.pgcoord.querier: got agent update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.427 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.427 [debu] coord1.pgcoord.querier: got agent update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.427 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.433 [debu] coord3.pgcoord.querier: queried agents coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_agents=2 error=<nil> t.go:85: 2023-08-29 22:55:58.434 [debu] coord3.pgcoord: wrote nodes coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":2,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.434 [debu] coord2.pgcoord: exiting recvLoop coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 error=EOF t.go:85: 2023-08-29 22:55:58.434 [debu] coord2.pgcoord: done sending updates coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa name=agent2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.440 [debu] coord3.pgcoord.querier: got agent update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.440 [debu] coord2.pgcoord: deleted agent binding coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 error=<nil> t.go:85: 2023-08-29 22:55:58.440 [debu] coord1.pgcoord.querier: got agent update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.440 [debu] coord2.pgcoord.querier: got agent update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.440 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.440 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.450 [debu] coord2.pgcoord: sent heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:58.450 [debu] coord2.pgcoord: ignoring our own heartbeat coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:58.450 [debu] coord3.pgcoord: got heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:58.450 [debu] coord1.pgcoord: got heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 other_coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:58.450 [debu] coord3.pgcoord: computed oldest heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 oldest="2023-08-29T22:55:58.41[336](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:337)0825Z" time_to_expiry=5.962518186s t.go:85: 2023-08-29 22:55:58.450 [debu] coord1.pgcoord: computed oldest heartbeat coordinator_id=a1c998d5-c3a2-4bed-a413-9ed7[364](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:365)1f0b2 oldest="2023-08-29T22:55:56.527511734Z" time_to_expiry=4.076648666s t.go:85: 2023-08-29 22:55:58.453 [debu] coord3.pgcoord.querier: queried agents coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_agents=1 error=<nil> t.go:85: 2023-08-29 22:55:58.454 [debu] coord1.pgcoord: got node update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node={"id":0,"as_of":"0001-01-01T00:00:00Z","key":"nodekey:0000000000000000000000000000000000000000000000000000000000000000","disco":"discokey:0000000000000000000000000000000000000000000000000000000000000000","preferred_derp":11,"derp_latency":null,"derp_forced_websockets":null,"addresses":null,"allowed_ips":null,"endpoints":null} t.go:85: 2023-08-29 22:55:58.454 [debu] coord3.pgcoord: wrote nodes coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":1,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.456 [debu] coord3.pgcoord.querier: got agent update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.456 [debu] coord1.pgcoord.querier: got agent update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.456 [debu] coord1.pgcoord: upserted agent binding coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 node="{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":11,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}" error=<nil> t.go:85: 2023-08-29 22:55:58.456 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.456 [debu] coord2.pgcoord.querier: got agent update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.456 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.461 [debu] coord3.pgcoord.querier: queried agents coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_agents=1 error=<nil> t.go:85: 2023-08-29 22:55:58.461 [debu] coord3.pgcoord: got node update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 node={"id":0,"as_of":"0001-01-01T00:00:00Z","key":"nodekey:0000000000000000000000000000000000000000000000000000000000000000","disco":"discokey:0000000000000000000000000000000000000000000000000000000000000000","preferred_derp":31,"derp_latency":null,"derp_forced_websockets":null,"addresses":null,"allowed_ips":null,"endpoints":null} t.go:85: 2023-08-29 22:55:58.461 [debu] coord3.pgcoord: wrote nodes coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":11,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.465 [debu] coord3.pgcoord: upserted client binding coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 node="{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":31,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}" error=<nil> t.go:85: 2023-08-29 22:55:58.471 [debu] coord3.pgcoord.querier: got client update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.471 [debu] coord3.pgcoord.querier: ignoring update because we have no mapper coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.471 [debu] coord1.pgcoord.querier: got client update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.471 [debu] coord2.pgcoord.querier: got client update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.471 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.474 [debu] coord1.pgcoord.querier: queried clients of agent coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 num_clients=1 error=<nil> t.go:85: 2023-08-29 22:55:58.474 [debu] coord1.pgcoord: exiting recvLoop coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 error=EOF t.go:85: 2023-08-29 22:55:58.474 [debu] coord1.pgcoord: wrote nodes coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 nodes="[{\"id\":0,\"as_of\":\"0001-01-01T00:00:00Z\",\"key\":\"nodekey:0000000000000000000000000000000000000000000000000000000000000000\",\"disco\":\"discokey:0000000000000000000000000000000000000000000000000000000000000000\",\"preferred_derp\":31,\"derp_latency\":null,\"derp_forced_websockets\":null,\"addresses\":null,\"allowed_ips\":null,\"endpoints\":null}]" t.go:85: 2023-08-29 22:55:58.474 [debu] coord1.pgcoord: unable to extend write deadline coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 name=agent1 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 error="io: read/write on closed pipe" t.go:85: 2023-08-29 22:55:58.474 [debu] coord3.pgcoord: exiting recvLoop coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 error=EOF t.go:85: 2023-08-29 22:55:58.474 [debu] coord3.pgcoord: done sending updates coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 t.go:85: 2023-08-29 22:55:58.478 [debu] coord1.pgcoord: deleted agent binding coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 error=<nil> t.go:85: 2023-08-29 22:55:58.479 [debu] coord3.pgcoord.querier: got agent update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.479 [debu] coord3.pgcoord.querier: ignoring update because we have no mapper coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.479 [debu] coord1.pgcoord.querier: got agent update coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.479 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.479 [debu] coord2.pgcoord.querier: got agent update coordinator_id=1b08773f-05b2-[401](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:402)3-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.479 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.481 [debu] coord3.pgcoord: deleted client binding coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 error=<nil> t.go:85: 2023-08-29 22:55:58.487 [debu] coord1.pgcoord.querier: got client update coordinator_id=a1c998d5-c3a2-4bed-a[413](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:414)-9ed73641f0b2 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.487 [debu] coord1.pgcoord.querier: ignoring update because we have no mapper coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.487 [debu] coord2.pgcoord.querier: got client update coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.487 [debu] coord2.pgcoord.querier: ignoring update because we have no mapper coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-5[417](https://github.com/coder/coder/actions/runs/6018353826/job/16326384384?pr=9414#step:5:418)2223bb71 t.go:85: 2023-08-29 22:55:58.487 [debu] coord3.pgcoord.querier: got client update coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.487 [debu] coord3.pgcoord.querier: ignoring update because we have no mapper coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 client_id=51c4addb-8e4c-4906-adc8-17fd57d480f9 agent_id=c2d6a785-f9fa-4698-80ed-54172223bb71 t.go:85: 2023-08-29 22:55:58.528 [info] coord3.pgcoord: closing coordinator coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 t.go:85: 2023-08-29 22:55:58.528 [info] coord2.pgcoord: closing coordinator coordinator_id=1b08773f-05b2-4013-91f0-055670bab7aa t.go:85: 2023-08-29 22:55:58.528 [info] coord1.pgcoord: closing coordinator coordinator_id=a1c998d5-c3a2-4bed-a413-9ed73641f0b2 t.go:90: 2023-08-29 22:55:58.528 [erro] coord3.pgcoord: failed to send heartbeat coordinator_id=22f4c8c3-b58f-4ffb-9d16-1276f2925a41 error="context canceled"