Skip to content
This repository was archived by the owner on Aug 23, 2020. It is now read-only.

Feat: added config variables for local snapshots #981

Merged
GalRogozinski merged 1 commit into from
Sep 13, 2018
Merged

Feat: added config variables for local snapshots #981

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions src/main/java/com/iota/iri/conf/BaseIotaConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,15 @@ public abstract class BaseIotaConfig implements IotaConfig {
protected double alpha = Defaults.ALPHA;
private int maxAnalyzedTransactions = Defaults.MAX_ANALYZED_TXS;

//Snapshot
protected boolean localSnapshotsEnabled = Defaults.LOCAL_SNAPSHOTS_ENABLED;
protected boolean localSnapshotsPruningEnabled = Defaults.LOCAL_SNAPSHOTS_PRUNING_ENABLED;
protected int localSnapshotsPruningDelay = Defaults.LOCAL_SNAPSHOTS_PRUNING_DELAY;
protected int localSnapshotsIntervalSynced = Defaults.LOCAL_SNAPSHOTS_INTERVAL_SYNCED;
protected int localSnapshotsIntervalUnsynced = Defaults.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED;
protected int localSnapshotsDepth = Defaults.LOCAL_SNAPSHOTS_DEPTH;
protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH;

public BaseIotaConfig() {
//empty constructor
}
Expand Down Expand Up @@ -453,6 +462,83 @@ protected void setpPropagateRequest(double pPropagateRequest) {
this.pPropagateRequest = pPropagateRequest;
}

@Override
public boolean getLocalSnapshotsEnabled() {
return this.localSnapshotsEnabled;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_ENABLED)
protected void setLocalSnapshotsEnabled(boolean localSnapshotsEnabled) {
this.localSnapshotsEnabled = localSnapshotsEnabled;
}

@Override
public boolean getLocalSnapshotsPruningEnabled() {
return this.localSnapshotsPruningEnabled;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-pruning-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_ENABLED)
protected void setLocalSnapshotsPruningEnabled(boolean localSnapshotsPruningEnabled) {
this.localSnapshotsPruningEnabled = localSnapshotsPruningEnabled;
}

@Override
public int getLocalSnapshotsPruningDelay() {
return this.localSnapshotsPruningDelay;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-pruning-delay"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_DELAY)
protected void setLocalSnapshotsPruningDelay(int localSnapshotsPruningDelay) {
this.localSnapshotsPruningDelay = localSnapshotsPruningDelay;
}

@Override
public int getLocalSnapshotsIntervalSynced() {
return this.localSnapshotsIntervalSynced;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-interval-synced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_SYNCED)
protected void setLocalSnapshotsIntervalSynced(int localSnapshotsIntervalSynced) {
this.localSnapshotsIntervalSynced = localSnapshotsIntervalSynced;
}

@Override
public int getLocalSnapshotsIntervalUnsynced() {
return this.localSnapshotsIntervalUnsynced;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-interval-unsynced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED)
protected void setLocalSnapshotsIntervalUnsynced(int localSnapshotsIntervalUnsynced) {
this.localSnapshotsIntervalUnsynced = localSnapshotsIntervalUnsynced;
}

@Override
public int getLocalSnapshotsDepth() {
return this.localSnapshotsDepth;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-depth"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_DEPTH)
protected void setLocalSnapshotsDepth(int localSnapshotsDepth) {
this.localSnapshotsDepth = localSnapshotsDepth;
}

@Override
public String getLocalSnapshotsBasePath() {
return this.localSnapshotsBasePath;
}

@JsonProperty
@Parameter(names = {"--local-snapshots-base-path"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_BASE_PATH)
protected void setLocalSnapshotsBasePath(String localSnapshotsBasePath) {
this.localSnapshotsBasePath = localSnapshotsBasePath;
}

@Override
public long getSnapshotTime() {
return Defaults.GLOBAL_SNAPSHOT_TIME;
Expand Down Expand Up @@ -669,6 +755,13 @@ public interface Defaults {
"KPWCHICGJZXKE9GSUDXZYUAPLHAKAHYHDXNPHENTERYMMBQOPSQIDENXKLKCEYCPVTZQLEEJVYJZV9BWU";

//Snapshot
boolean LOCAL_SNAPSHOTS_ENABLED = true;
boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false;
int LOCAL_SNAPSHOTS_PRUNING_DELAY = 1000;
int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10;
int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 5000;
String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet";
Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we actually need this configuration variable? This should be automatically be adjusted depending to the network you are using. This way we can also always rely on a standard location for local snapshot files.

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

since the file is currently persisted in the filesystem i would want to give the user the option to have control over the location of the file.

It might for example be possible that people run IOTA nodes with the folder of the binary being mounted as read only to secure the iri.jar against manipulations. In this case you might want to give a path to another directory where you store the working data like the db or the snapshot files.

Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already allow users to specify a data directory for their IRI nodes, I believe the local snapshots should be just dropped into that same folder; without giving the user this option, which I don't find the use for.

Copy link
Copy Markdown
Contributor

@alon-e alon-e Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the DB/persistable folder structure should be revisited, but as a different issue IMO.

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we create issues for minor things like this so i can change it afterwards? because once we have it merged i can more easily work on changes since i don't have to do them in 10 different branches.

Copy link
Copy Markdown
Contributor

@jakubcech jakubcech Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, please, create an issue for this.

int LOCAL_SNAPSHOTS_DEPTH = 500;
Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we have any sort of "story" backing up these default values?

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10
Taking local snapshots regularly makes the processing faster since it doesnt have to examine so many new transactions so we want to do it relatively often but also not too often since it still consumes as few resources - 5 minutes have proven to be a good value while testing to keep the processing times short (below a few seconds) and still don't bother the node too many times with creating the snapshots.

LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 5000
When we are syncing we usually process a lot of milestones very rapidly. To not bother the syncing process too much but still have a fast resume point when IRI crashes for example, we create a snapshot every 5000 milestones. In addition to reducing the amount of local snapshots taken and the corresponding load on the node, it also creates a situation where we have a bigger chunk of the old tangle data available most of the time because the transactions only get pruned when a snapshot was taken. This allows us to revert the state of the ledger further back in case we detect a milestone transaction that was skipped previously and continue the syncing without an unrecoverable error - repairing milestones will be included in a future PR.

int LOCAL_SNAPSHOTS_DEPTH = 500;
I reduced the snapshot depth back to 500 because it works 99.9% of the time (i only once faced a situation where data was referenced by the coo that was older than 500 milestones. Since we also have a pruning delay now (new feature) which keeps old transactions prior to the taken snapshot., this situation should not be a problem anymore. At the same time a lower snapshot depth allows it to sync new nodes extremely fast.

Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we also have a pruning delay now (new feature) which keeps old transactions prior to the taken snapshot.

So effectively you will be able to reference data up to LOCAL_SNAPSHOTS_DEPTH + LOCAL_SNAPSHOTS_PRUNING_DELAY milestones back, assuming pruning is enabled.

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep

String SNAPSHOT_FILE = "/snapshotMainnet.txt";
String SNAPSHOT_SIG_FILE = "/snapshotMainnet.sig";
String PREVIOUS_EPOCHS_SPENT_ADDRESSES_TXT = "/previousEpochsSpentAddresses.txt";
Expand Down
42 changes: 42 additions & 0 deletions src/main/java/com/iota/iri/conf/SnapshotConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,36 @@
*/
public interface SnapshotConfig extends Config {

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_ENABLED}
*/
boolean getLocalSnapshotsEnabled();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_PRUNING_ENABLED}
*/
boolean getLocalSnapshotsPruningEnabled();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_PRUNING_DELAY}
*/
int getLocalSnapshotsPruningDelay();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_INTERVAL_SYNCED}
*/
int getLocalSnapshotsIntervalSynced();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED}
*/
int getLocalSnapshotsIntervalUnsynced();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_DEPTH}
*/
int getLocalSnapshotsDepth();

/**
* @return {@value Descriptions#SNAPSHOT_TIME}
*/
Expand All @@ -25,6 +55,11 @@ public interface SnapshotConfig extends Config {
*/
int getMilestoneStartIndex();

/**
* @return {@value Descriptions#LOCAL_SNAPSHOTS_BASE_PATH}
*/
String getLocalSnapshotsBasePath();

/**
* @return {@value Descriptions#NUMBER_OF_KEYS_IN_A_MILESTONE}
*/
Expand All @@ -42,6 +77,13 @@ public interface SnapshotConfig extends Config {

interface Descriptions {

String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled.";
String LOCAL_SNAPSHOTS_PRUNING_ENABLED = "Flag that determines if pruning of old data is enabled.";
String LOCAL_SNAPSHOTS_PRUNING_DELAY = "Only prune data that precedes the local snapshot by n milestones.";
String LOCAL_SNAPSHOTS_INTERVAL_SYNCED = "Take local snapshots every n milestones if the node is fully synced.";
String LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = "Take local snapshots every n milestones if the node is syncing.";
String LOCAL_SNAPSHOTS_DEPTH = "Number of milestones to keep.";
Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this the number of milestones to keep prior to latest local snapshot? Can you elaborate the description a bit?

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes - if the milestone was taken with a depth of 500 and we have a pruning delay that is set to 5000, then we only prune data that is 5500 milestones in the past. This allows us to create local snapshot files that are suitable for very fast syncs (new nodes) and still maintain enough old data to never run into issues with the coordinator suddenly approving stuff that is really old.

Copy link
Copy Markdown
Contributor

@karimodm karimodm Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, that's clear; just to be more verbose in the doc.

Copy link
Copy Markdown
Contributor

@alon-e alon-e Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that this description should be more informative.
what are the ranges, what do they mean for the users?

Copy link
Copy Markdown
Author

@hmoog hmoog Sep 12, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe also create a separate issue after merging so we can get these changes in first - its kind of tricky to break the whole shit down anyway

String LOCAL_SNAPSHOTS_BASE_PATH = "Path to the snapshot files (without file extensions).";
String SNAPSHOT_TIME = "Epoch time of the last snapshot.";
String SNAPSHOT_FILE = "Path of the file that contains the state of the ledger at the last snapshot.";
String SNAPSHOT_SIGNATURE_FILE = "Path to the file that contains a signature for the snapshot file.";
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/iota/iri/conf/TestnetConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class TestnetConfig extends BaseIotaConfig {
protected int numberOfKeysInMilestone = Defaults.KEYS_IN_MILESTONE;
protected int transactionPacketSize = Defaults.PACKET_SIZE;
protected int requestHashSize = Defaults.REQUEST_HASH_SIZE;
protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH;

public TestnetConfig() {
super();
Expand Down Expand Up @@ -161,6 +162,7 @@ public void setDbLogPath(String dbLogPath) {
public interface Defaults {
String COORDINATOR_ADDRESS = "EQQFCZBIHRHWPXKMTOLMYUYPCN9XLMJPYZVFJSAY9FQHCCLWTOLLUGKKMXYFDBOOYFBLBI9WUEILGECYM";
boolean VALIDATE_MILESTONE_SIG = true;
String LOCAL_SNAPSHOTS_BASE_PATH = "testnet";
String SNAPSHOT_FILE = "/snapshotTestnet.txt";
int REQUEST_HASH_SIZE = 49;
String SNAPSHOT_SIG = "/snapshotTestnet.sig";
Expand Down