Implement -reindex option

This PR adds to the GRC::Upgrade utility class and implements the wallet
-reindex startup parameter.

Author: jamescowens

src/gridcoin/upgrade.cpp

@@ -485,9 +485,9 @@ void Upgrade::VerifySHA256SUM()
     }
 }
 
-void Upgrade::CleanupBlockchainData()
+bool Upgrade::GetActualCleanupPath(fs::path& actual_cleanup_path)
 {
-    fs::path CleanupPath = GetDataDir();
+    actual_cleanup_path = GetDataDir();
 
     // This is required because of problems with junction point handling in the boost filesystem library. Please see
     // https://github.com/boostorg/filesystem/issues/125. We are not quite ready to switch over to std::filesystem yet.
@@ -496,16 +496,16 @@ void Upgrade::CleanupBlockchainData()
     //
     // I don't believe it is very common for Windows users to redirect the Gridcoin data directory with a junction point,
     // but it is certainly possible. We should handle it as gracefully as possible.
-    if (fs::is_symlink(CleanupPath))
+    if (fs::is_symlink(actual_cleanup_path))
     {
         LogPrintf("INFO: %s: Data directory is a symlink.",
                   __func__);
 
         try
         {
-            LogPrintf("INFO: %s: True path for the symlink is %s.", __func__, fs::read_symlink(CleanupPath).string());
+            LogPrintf("INFO: %s: True path for the symlink is %s.", __func__, fs::read_symlink(actual_cleanup_path).string());
 
-            CleanupPath = fs::read_symlink(CleanupPath);
+            actual_cleanup_path = fs::read_symlink(actual_cleanup_path);
         }
         catch (fs::filesystem_error &ex)
         {
@@ -515,10 +515,19 @@ void Upgrade::CleanupBlockchainData()
 
             DownloadStatus.SetCleanupBlockchainDataFailed(true);
 
-            return;
+            return false;
         }
     }
 
+    return true;
+}
+
+void Upgrade::CleanupBlockchainData(bool include_blockchain_data_files)
+{
+    fs::path CleanupPath;
+
+    if (!GetActualCleanupPath(CleanupPath)) return;
+
     unsigned int total_items = 0;
     unsigned int items = 0;
 
@@ -559,7 +568,7 @@ void Upgrade::CleanupBlockchainData()
                 continue;
             }
 
-            else if (fs::is_regular_file(*Iter))
+            else if (fs::is_regular_file(*Iter) && include_blockchain_data_files)
             {
                 size_t FileLoc = Iter->path().filename().string().find("blk");
 
@@ -648,7 +657,7 @@ void Upgrade::CleanupBlockchainData()
                 continue;
             }
 
-            else if (fs::is_regular_file(*Iter))
+            else if (fs::is_regular_file(*Iter) && include_blockchain_data_files)
             {
                 size_t FileLoc = Iter->path().filename().string().find("blk");
 
@@ -875,13 +884,126 @@ void Upgrade::DeleteSnapshot()
     }
 }
 
-bool Upgrade::ResetBlockchainData()
+bool Upgrade::ResetBlockchainData(bool include_blockchain_data_files)
 {
-    CleanupBlockchainData();
+    CleanupBlockchainData(include_blockchain_data_files);
 
     return (DownloadStatus.GetCleanupBlockchainDataComplete() && !DownloadStatus.GetCleanupBlockchainDataFailed());
 }
 
+bool Upgrade::MoveBlockDataFiles(std::set<std::pair<fs::path, uintmax_t>>& block_data_files)
+{
+    fs::path cleanup_path;
+
+    if (!GetActualCleanupPath(cleanup_path)) return false;
+
+    fs::directory_iterator IterEnd;
+
+    try {
+        for (fs::directory_iterator Iter(cleanup_path); Iter != IterEnd; ++Iter) {
+            if (fs::is_regular_file(*Iter)) {
+                size_t FileLoc = Iter->path().filename().string().find("blk");
+
+                if (FileLoc != std::string::npos) {
+                    std::string filetocheck = Iter->path().filename().string();
+
+                    // Check it ends with .dat and starts with blk
+                    if (filetocheck.substr(0, 3) == "blk" && filetocheck.substr(filetocheck.length() - 4, 4) == ".dat") {
+                        fs::path new_name = *Iter;
+                        new_name.replace_extension(".dat.orig");
+
+                        uintmax_t file_size = fs::file_size(Iter->path());
+
+                        // Rename with orig as the extension, because ProcessBlock will load blocks into a new block data
+                        // file.
+                        fs::rename(*Iter, new_name);
+                        block_data_files.insert(std::make_pair(new_name, file_size));
+                    }
+                }
+            }
+        }
+    } catch (fs::filesystem_error &ex) {
+        error("%s: Exception occurred: %s. Failed to rename block data files to blk*.dat.orig in preparation for "
+              "reindexing.", __func__, ex.what());
+
+        return false;
+    }
+
+    return true;
+}
+
+bool Upgrade::ReindexBlockchainData(std::set<std::pair<fs::path, uintmax_t>>& block_data_files)
+{
+    bool successful = true;
+
+    uintmax_t total_size = 0;
+    uintmax_t cumulative_size = 0;
+
+    for (const auto& iter : block_data_files) {
+        total_size += iter.second;
+    }
+
+    if (!total_size) return false;
+
+    try {
+        for (const auto& iter : block_data_files) {
+
+            unsigned int percent_start = cumulative_size * (uintmax_t) 100 / total_size;
+
+            cumulative_size += iter.second;
+
+            unsigned int percent_end = cumulative_size * (uintmax_t) 100 / total_size;
+
+            FILE *block_data_file = fsbridge::fopen(iter.first, "rb");
+
+            LogPrintf("INFO: %s: Loading blocks from %s.", __func__, iter.first.filename().string());
+
+            if (!LoadExternalBlockFile(block_data_file, iter.second, percent_start, percent_end)) {
+                successful = false;
+
+                break;
+            }
+        }
+    } catch (fs::filesystem_error &ex) {
+        error("%s: Exception occurred: %s. Failure occurred during attempt to load blocks from original "
+              "block data file(s).", __func__, ex.what());
+
+        successful = false;
+    }
+
+    if (successful) {
+        try {
+            fs::path cleanup_path;
+
+            if (!GetActualCleanupPath(cleanup_path)) return false;
+
+            for (const auto& iter : block_data_files) {
+                if (!fs::remove(iter.first)) {
+                    LogPrintf("WARN: %s: Reindexing of the blockchain was successful; however, one or more of "
+                              "the original block data files (%s) was not able to be deleted. You "
+                              "will have to delete this file manually.", __func__, iter.first.filename().string());
+                }
+            }
+        }
+        catch (fs::filesystem_error &ex) {
+            LogPrintf("WARN: %s: Exception occurred: %s. This error occurred while attempting to delete the original block "
+                      "data files (blk*.dat.orig). You will have to delete these manually.", __func__, ex.what());
+        }
+    } else {
+        error("%s: A failure occurred during the reindexing of the block data files. The blockchain state is invalid and "
+              "you should restart the wallet with the -resetblockchaindata option to clear out the blockchain database "
+              "and re-sync the blockchain from the network.", __func__);
+
+        DownloadStatus.SetCleanupBlockchainDataFailed(true);
+
+        return false;
+    }
+
+    LogPrintf("INFO: %s: Reindex of the blockchain data was successful.", __func__);
+
+    return true;
+}
+
 std::string Upgrade::ResetBlockchainMessages(ResetBlockchainMsg _msg)
 {
     std::stringstream stream;

src/gridcoin/upgrade.h

@@ -10,6 +10,7 @@
 #include <sstream>
 #include <iomanip>
 #include <vector>
+#include <set>
 
 #include "gridcoin/scraper/http.h"
 #include "node/ui_interface.h"
@@ -140,12 +141,19 @@ class Upgrade
     //!
     static void DownloadSnapshot();
 
+    //!
+    //! \brief Resolves symlinks to the actual path.
+    //! \param actual_cleanup_path is the resolved path
+    //! \return
+    //!
+    static bool GetActualCleanupPath(fs::path& actual_cleanup_path);
+
     //!
     //! \brief Cleans up previous blockchain data if any is found
     //!
     //! \return Bool on the success of cleanup
     //!
-    static void CleanupBlockchainData();
+    static void CleanupBlockchainData(bool include_blockchain_data_files = true);
 
     //!
     //! \brief This is the worker thread "main" that actually does the brunt of the snapshot download and extraction work.
@@ -183,8 +191,20 @@ class Upgrade
     //!
     //! \returns Bool on the success of blockchain cleanup
     //!
-    static bool ResetBlockchainData();
+    static bool ResetBlockchainData(bool include_blockchain_data_files = true);
+
+    //!
+    //! \brief Moves the block data files from .dat to .dat.orig in preparation for reindexing.
+    //! \return
+    //!
+    static bool MoveBlockDataFiles(std::set<std::pair<fs::path, uintmax_t> > &block_data_files);
 
+    //!
+    //! \brief Utility function to support the -reindex startup parameter to rebuild txleveldb and accrual from
+    //! existing blockchain data files.
+    //! \return
+    //!
+    static bool ReindexBlockchainData(std::set<std::pair<fs::path, uintmax_t> > &block_data_files);
     //!
     //! \brief Small function to return translated messages.
     //!