Skip to content

Commit 5fa2ed3

Browse files
original-brownbearoriginal-brownbear
authored
Fix CCR following a datastream with closed indices on the follower corrupting the datastream (#87076) (#87083)
Reproducer and fix for #87048. Reproduces the edge case by closing follower index that is part of a datastream and then recreating and re-adding that same index on the leader to make it get picked up by the auto-follower again. Using stats call in the test mainly to reproduce the exact issue that motivated #87048 and to show that the datastream is correctly resolved by the index name expression resolver. closes #87048
1 parent b5565a6 commit 5fa2ed3

9 files changed

Lines changed: 149 additions & 14 deletions

File tree

‎docs/changelog/87076.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pr: 87076
2+
summary: Fix CCR following a datastream with closed indices on the follower corrupting
3+
the datastream
4+
area: "CCR"
5+
type: bug
6+
issues:
7+
- 87048

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,14 @@
3535
import java.time.Instant;
3636
import java.util.ArrayList;
3737
import java.util.Collection;
38-
import java.util.Collections;
3938
import java.util.Comparator;
4039
import java.util.HashMap;
40+
import java.util.HashSet;
4141
import java.util.List;
4242
import java.util.Locale;
4343
import java.util.Map;
4444
import java.util.Objects;
45+
import java.util.Set;
4546
import java.util.function.Function;
4647
import java.util.function.LongSupplier;
4748

@@ -111,7 +112,7 @@ public DataStream(
111112
IndexMode indexMode
112113
) {
113114
this.name = name;
114-
this.indices = Collections.unmodifiableList(indices);
115+
this.indices = List.copyOf(indices);
115116
this.generation = generation;
116117
this.metadata = metadata;
117118
assert system == false || hidden; // system indices must be hidden
@@ -121,7 +122,17 @@ public DataStream(
121122
this.system = system;
122123
this.allowCustomRouting = allowCustomRouting;
123124
this.indexMode = indexMode;
125+
assert assertConsistent(this.indices);
126+
}
127+
128+
private static boolean assertConsistent(List<Index> indices) {
124129
assert indices.size() > 0;
130+
final Set<String> indexNames = new HashSet<>();
131+
for (Index index : indices) {
132+
final boolean added = indexNames.add(index.getName());
133+
assert added : "found duplicate index entries in " + indices;
134+
}
135+
return true;
125136
}
126137

127138
public String getName() {

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
public class DataStreamMetadata implements Metadata.Custom {
3333

3434
public static final String TYPE = "data_stream";
35+
36+
public static final DataStreamMetadata EMPTY = new DataStreamMetadata(Map.of(), Map.of());
3537
private static final ParseField DATA_STREAM = new ParseField("data_stream");
3638
private static final ParseField DATA_STREAM_ALIASES = new ParseField("data_stream_aliases");
3739
@SuppressWarnings("unchecked")

‎server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1540,6 +1540,10 @@ public Builder put(DataStream dataStream) {
15401540
return this;
15411541
}
15421542

1543+
public DataStreamMetadata dataStreamMetadata() {
1544+
return (DataStreamMetadata) this.customs.getOrDefault(DataStreamMetadata.TYPE, DataStreamMetadata.EMPTY);
1545+
}
1546+
15431547
public boolean put(String aliasName, String dataStream, Boolean isWriteDataStream, String filter) {
15441548
previousIndicesLookup = null;
15451549

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@
8484
public class MetadataIndexTemplateService {
8585

8686
public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp";
87-
private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING;
87+
public static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING;
8888

8989
private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING;
9090

‎x-pack/plugin/ccr/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ dependencies {
3434
testImplementation(testArtifact(project(xpackModule('core'))))
3535
testImplementation(testArtifact(project(xpackModule('monitoring'))))
3636
testImplementation(project(":modules:analysis-common"))
37+
testImplementation(project(":modules:data-streams"))
3738
}
3839

3940
tasks.named("testingConventions").configure {

‎x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,27 @@
1010
import org.elasticsearch.ElasticsearchException;
1111
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
1212
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
13+
import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
14+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
15+
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
16+
import org.elasticsearch.action.datastreams.CreateDataStreamAction;
17+
import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
1318
import org.elasticsearch.client.internal.Client;
19+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
20+
import org.elasticsearch.cluster.metadata.DataStream;
21+
import org.elasticsearch.cluster.metadata.DataStreamAction;
1422
import org.elasticsearch.cluster.metadata.IndexMetadata;
1523
import org.elasticsearch.cluster.metadata.Metadata;
24+
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
25+
import org.elasticsearch.cluster.metadata.Template;
1626
import org.elasticsearch.common.Strings;
1727
import org.elasticsearch.common.regex.Regex;
1828
import org.elasticsearch.common.settings.Settings;
1929
import org.elasticsearch.common.unit.ByteSizeUnit;
2030
import org.elasticsearch.common.unit.ByteSizeValue;
2131
import org.elasticsearch.core.CheckedRunnable;
2232
import org.elasticsearch.core.TimeValue;
33+
import org.elasticsearch.datastreams.DataStreamsPlugin;
2334
import org.elasticsearch.index.IndexNotFoundException;
2435
import org.elasticsearch.indices.SystemIndexDescriptor;
2536
import org.elasticsearch.plugins.Plugin;
@@ -51,8 +62,10 @@
5162
import java.util.stream.Stream;
5263

5364
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
65+
import static org.hamcrest.Matchers.aMapWithSize;
5466
import static org.hamcrest.Matchers.equalTo;
5567
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
68+
import static org.hamcrest.Matchers.hasKey;
5669
import static org.hamcrest.Matchers.hasSize;
5770
import static org.hamcrest.Matchers.is;
5871
import static org.hamcrest.Matchers.notNullValue;
@@ -67,7 +80,7 @@ protected boolean reuseClusters() {
6780

6881
@Override
6982
protected Collection<Class<? extends Plugin>> nodePlugins() {
70-
return Stream.concat(super.nodePlugins().stream(), Stream.of(FakeSystemIndex.class)).collect(Collectors.toList());
83+
return Stream.concat(super.nodePlugins().stream(), Stream.of(FakeSystemIndex.class, DataStreamsPlugin.class)).toList();
7184
}
7285

7386
public static class FakeSystemIndex extends Plugin implements SystemIndexPlugin {
@@ -621,6 +634,98 @@ public void testAutoFollowExclusion() throws Exception {
621634
assertFalse(ESIntegTestCase.indexExists("copy-logs-201801", followerClient()));
622635
}
623636

637+
public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception {
638+
final String datastream = "logs-1";
639+
PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("template-id");
640+
request.indexTemplate(
641+
new ComposableIndexTemplate(
642+
List.of("logs-*"),
643+
new Template(
644+
Settings.builder()
645+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
646+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
647+
.build(),
648+
null,
649+
null
650+
),
651+
null,
652+
null,
653+
null,
654+
null,
655+
new ComposableIndexTemplate.DataStreamTemplate(),
656+
null
657+
)
658+
);
659+
assertAcked(leaderClient().execute(PutComposableIndexTemplateAction.INSTANCE, request).get());
660+
661+
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(datastream);
662+
assertAcked(leaderClient().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get());
663+
leaderClient().prepareIndex(datastream)
664+
.setCreate(true)
665+
.setSource("foo", "bar", DataStream.TIMESTAMP_FIELD.getName(), randomNonNegativeLong())
666+
.get();
667+
668+
PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request();
669+
followRequest.setName("pattern-1");
670+
followRequest.setRemoteCluster("leader_cluster");
671+
followRequest.setLeaderIndexPatterns(List.of("logs-*"));
672+
followRequest.setFollowIndexNamePattern("{{leader_index}}");
673+
assertTrue(followerClient().execute(PutAutoFollowPatternAction.INSTANCE, followRequest).get().isAcknowledged());
674+
675+
logger.info("--> roll over once and wait for the auto-follow to pick up the new index");
676+
leaderClient().admin().indices().prepareRolloverIndex("logs-1").get();
677+
assertLongBusy(() -> {
678+
AutoFollowStats autoFollowStats = getAutoFollowStats();
679+
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(1L));
680+
});
681+
682+
ensureFollowerGreen("*");
683+
684+
final RolloverResponse rolloverResponse = leaderClient().admin().indices().prepareRolloverIndex(datastream).get();
685+
final String indexInDatastream = rolloverResponse.getOldIndex();
686+
687+
logger.info("--> closing [{}] on follower so it will be re-opened by crr", indexInDatastream);
688+
assertAcked(followerClient().admin().indices().prepareClose(indexInDatastream).setMasterNodeTimeout(TimeValue.MAX_VALUE).get());
689+
690+
logger.info("--> deleting and recreating index [{}] on leader to change index uuid on leader", indexInDatastream);
691+
assertAcked(leaderClient().admin().indices().prepareDelete(indexInDatastream).get());
692+
assertAcked(
693+
leaderClient().admin()
694+
.indices()
695+
.prepareCreate(indexInDatastream)
696+
.setMapping(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_MAPPING.toString())
697+
.get()
698+
);
699+
leaderClient().prepareIndex(indexInDatastream)
700+
.setCreate(true)
701+
.setSource("foo", "bar", DataStream.TIMESTAMP_FIELD.getName(), randomNonNegativeLong())
702+
.get();
703+
leaderClient().execute(
704+
ModifyDataStreamsAction.INSTANCE,
705+
new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(datastream, indexInDatastream)))
706+
).get();
707+
708+
assertLongBusy(() -> {
709+
AutoFollowStats autoFollowStats = getAutoFollowStats();
710+
assertThat(autoFollowStats.getNumberOfSuccessfulFollowIndices(), equalTo(3L));
711+
});
712+
713+
final Metadata metadata = followerClient().admin().cluster().prepareState().get().getState().metadata();
714+
final DataStream dataStream = metadata.dataStreams().get(datastream);
715+
assertTrue(dataStream.getIndices().stream().anyMatch(i -> i.getName().equals(indexInDatastream)));
716+
assertEquals(IndexMetadata.State.OPEN, metadata.index(indexInDatastream).getState());
717+
ensureFollowerGreen("*");
718+
final IndicesStatsResponse stats = followerClient().admin().indices().prepareStats(datastream).get();
719+
assertThat(stats.getIndices(), aMapWithSize(2));
720+
721+
assertAcked(leaderClient().admin().indices().prepareDelete(indexInDatastream).get());
722+
assertAcked(followerClient().admin().indices().prepareDelete(indexInDatastream).setMasterNodeTimeout(TimeValue.MAX_VALUE).get());
723+
ensureFollowerGreen("*");
724+
final IndicesStatsResponse statsAfterDelete = followerClient().admin().indices().prepareStats(datastream).get();
725+
assertThat(statsAfterDelete.getIndices(), aMapWithSize(1));
726+
assertThat(statsAfterDelete.getIndices(), hasKey(rolloverResponse.getNewIndex()));
727+
}
728+
624729
private void putAutoFollowPatterns(String name, String[] patterns) {
625730
putAutoFollowPatterns(name, patterns, Collections.emptyList());
626731
}

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ protected void doRun() {
219219
} else {
220220
String followerIndexName = request.getFollowerIndex();
221221
BiConsumer<ClusterState, Metadata.Builder> updater = (currentState, mdBuilder) -> {
222-
DataStream localDataStream = currentState.getMetadata().dataStreams().get(remoteDataStream.getName());
222+
DataStream localDataStream = mdBuilder.dataStreamMetadata().dataStreams().get(remoteDataStream.getName());
223223
Index followerIndex = mdBuilder.get(followerIndexName).getIndex();
224224
assert followerIndex != null;
225225

@@ -329,14 +329,19 @@ static DataStream updateLocalDataStream(Index backingIndexToFollow, DataStream l
329329
);
330330
}
331331

332-
List<Index> backingIndices = new ArrayList<>(localDataStream.getIndices());
333-
backingIndices.add(backingIndexToFollow);
334-
335-
// When following an older backing index it should be positioned before the newer backing indices.
336-
// Currently the assumption is that the newest index (highest generation) is the write index.
337-
// (just appending an older backing index to the list of backing indices would break that assumption)
338-
// (string sorting works because of the naming backing index naming scheme)
339-
backingIndices.sort(Comparator.comparing(Index::getName));
332+
final List<Index> backingIndices;
333+
if (localDataStream.getIndices().contains(backingIndexToFollow) == false) {
334+
backingIndices = new ArrayList<>(localDataStream.getIndices());
335+
backingIndices.add(backingIndexToFollow);
336+
// When following an older backing index it should be positioned before the newer backing indices.
337+
// Currently the assumption is that the newest index (highest generation) is the write index.
338+
// (just appending an older backing index to the list of backing indices would break that assumption)
339+
// (string sorting works because of the naming backing index naming scheme)
340+
backingIndices.sort(Comparator.comparing(Index::getName));
341+
} else {
342+
// edge case where the index was closed on the follower and was already in the datastream's index list
343+
backingIndices = localDataStream.getIndices();
344+
}
340345

341346
return new DataStream(
342347
localDataStream.getName(),

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ public void testPerformActionSameOriginalTargetError() {
205205
.numberOfReplicas(randomIntBetween(0, 5))
206206
.build();
207207

208-
List<Index> backingIndices = List.of(sourceIndexMetadata.getIndex(), writeIndexMetadata.getIndex());
208+
List<Index> backingIndices = List.of(writeIndexMetadata.getIndex());
209209
ClusterState clusterState = ClusterState.builder(emptyClusterState())
210210
.metadata(
211211
Metadata.builder()

0 commit comments

Comments
 (0)