Flaky TestMultitenantAlertmanager_SyncOnRingTopologyChanges

[pracucci]

The TestMultitenantAlertmanager_SyncOnRingTopologyChanges is flaky. See this CI run:

level=info component=MultiTenantAlertmanager msg="instance not found in the ring" instance=instance-1 ring=alertmanager
level=info component=MultiTenantAlertmanager waitinguntilalertmanagerisJOININGinthering=(MISSING)
level=info component=MultiTenantAlertmanager msg="alertmanager is JOINING in the ring"
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=0
level=info component=MultiTenantAlertmanager msg="waiting until initial state sync is complete for all users"
level=info component=MultiTenantAlertmanager msg="initial state sync is complete"
level=info component=MultiTenantAlertmanager msg="waiting until alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="instance not found in the ring" instance=instance-2 ring=alertmanager
level=info component=MultiTenantAlertmanager waitinguntilalertmanagerisJOININGinthering=(MISSING)
level=info component=MultiTenantAlertmanager msg="alertmanager is JOINING in the ring"
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=0
level=info component=MultiTenantAlertmanager msg="waiting until initial state sync is complete for all users"
level=info component=MultiTenantAlertmanager msg="initial state sync is complete"
level=info component=MultiTenantAlertmanager msg="waiting until alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="instance not found in the ring" instance=instance-3 ring=alertmanager
level=info component=MultiTenantAlertmanager waitinguntilalertmanagerisJOININGinthering=(MISSING)
level=info component=MultiTenantAlertmanager msg="alertmanager is JOINING in the ring"
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=0
level=info component=MultiTenantAlertmanager msg="waiting until initial state sync is complete for all users"
level=info component=MultiTenantAlertmanager msg="initial state sync is complete"
level=info component=MultiTenantAlertmanager msg="waiting until alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="alertmanager is ACTIVE in the ring"
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=3
level=debug component=MultiTenantAlertmanager msg="setting config" user=user1
level=debug component=MultiTenantAlertmanager msg="initializing new per-tenant alertmanager" user=user1
level=debug component=MultiTenantAlertmanager user=user1 msg="starting tenant alertmanager with ring-based replication"
level=info component=MultiTenantAlertmanager user=user1 msg="Waiting for notification and silences to settle..."
level=debug component=MultiTenantAlertmanager msg="contacting replica for full state" user=user1 addr=127.0.0.1-3:0
level=info component=MultiTenantAlertmanager user=user1 msg="Waiting for notification and silences to settle..."
level=info component=MultiTenantAlertmanager user=user1 msg="skipping settling (no replicas)"
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=1
level=debug component=MultiTenantAlertmanager msg="setting config" user=user2
level=debug component=MultiTenantAlertmanager msg="initializing new per-tenant alertmanager" user=user2
level=debug component=MultiTenantAlertmanager user=user2 msg="starting tenant alertmanager with ring-based replication"
level=info component=MultiTenantAlertmanager user=user2 msg="Waiting for notification and silences to settle..."
level=info component=MultiTenantAlertmanager user=user2 msg="skipping settling (no replicas)"
level=debug component=MultiTenantAlertmanager user=user1 msg="persisting state" user=user1
level=debug component=MultiTenantAlertmanager user=user2 msg="persisting state" user=user2
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=1
level=debug component=MultiTenantAlertmanager msg="setting config" user=user1
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=1
level=debug component=MultiTenantAlertmanager msg="setting config" user=user2
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=0
level=info component=MultiTenantAlertmanager msg="deactivating per-tenant alertmanager" user=user1
level=debug component=MultiTenantAlertmanager user=user1 component=nflog msg="Running maintenance"
level=debug component=MultiTenantAlertmanager user=user1 component=silences msg="Running maintenance"
level=debug component=MultiTenantAlertmanager user=user1 component=nflog msg="Maintenance done" duration=2.034386ms size=0
level=debug component=MultiTenantAlertmanager user=user1 component=silences msg="Maintenance done" duration=1.628189ms size=0
level=info component=MultiTenantAlertmanager msg="deactivated per-tenant alertmanager" user=user1
level=info component=MultiTenantAlertmanager msg="deleted local directory for user" dir=/tmp/alertmanager364730707/user1 user=user1
level=info component=MultiTenantAlertmanager msg="deleted remote state for user" user=user1
level=info component=MultiTenantAlertmanager msg="synchronizing alertmanager configs for users"
level=debug component=MultiTenantAlertmanager msg="adding configurations" num_configs=1
level=debug component=MultiTenantAlertmanager msg="setting config" user=user2
level=debug component=MultiTenantAlertmanager user=user2 component=silences msg="Running maintenance"
level=info component=MultiTenantAlertmanager msg="unregistering instance from ring" ring=alertmanager
level=info component=MultiTenantAlertmanager msg="instance removed from the ring" ring=alertmanager
level=debug component=MultiTenantAlertmanager user=user2 component=nflog msg="Running maintenance"
level=debug component=MultiTenantAlertmanager user=user2 component=silences msg="Maintenance done" duration=1.576889ms size=0
level=debug component=MultiTenantAlertmanager user=user2 component=nflog msg="Maintenance done" duration=1.062792ms size=0
level=info component=MultiTenantAlertmanager msg="unregistering instance from ring" ring=alertmanager
level=info component=MultiTenantAlertmanager msg="instance removed from the ring" ring=alertmanager
--- FAIL: TestMultitenantAlertmanager_SyncOnRingTopologyChanges (2.71s)
    --- FAIL: TestMultitenantAlertmanager_SyncOnRingTopologyChanges/should_sync_when_an_healthy_instance_becomes_unhealthy (1.02s)
        multitenant_test.go:1330: expected 2, got 1
FAIL
FAIL	github.com/cortexproject/cortex/pkg/alertmanager	104.207s

/cc @stevesg

[bboreham]

This failure seems very similar: https://github.com/cortexproject/cortex/runs/3173838252