bitshares · abitmore · Jan 11, 2023 · Dec 4, 2022 · Dec 4, 2022 · Dec 4, 2022
diff --git a/.github/workflows/build-and-test.mac.yml b/.github/workflows/build-and-test.mac.yml
@@ -42,7 +42,7 @@ jobs:
       run: |
         export CCACHE_DIR="$GITHUB_WORKSPACE/ccache"
         mkdir -p "$CCACHE_DIR"
-        make -j 2 -C _build witness_node cli_wallet app_test cli_test chain_test
+        make -j 3 -C _build witness_node cli_wallet app_test cli_test chain_test
         df -h
     - name: Unit-Tests
       run: |

diff --git a/.github/workflows/build-and-test.ubuntu-debug.yml b/.github/workflows/build-and-test.ubuntu-debug.yml
@@ -8,12 +8,26 @@ jobs:
     name: Build and test in Debug mode
     strategy:
       matrix:
-        os: [ ubuntu-18.04, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env ES_JAVA_OPTS="-Xms512m -Xmx512m"
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
+      elasticsearch7:
+        image: elastic/elasticsearch:7.17.8
+        options: >-
+          --env ES_JAVA_OPTS="-Xms512m -Xmx512m"
+          --env discovery.type=single-node
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9201:9200
     steps:
     - name: Install dependencies
       run: |
@@ -51,11 +65,6 @@ jobs:
         pwd
         df -h .
         free
-        sudo dd if=/dev/zero of=/swapfile bs=1024 count=4M
-        sudo chmod 600 /swapfile
-        sudo mkswap /swapfile
-        sudo swapon /swapfile
-        free
         mkdir -p _build
         sudo mkdir -p /_build/libraries /_build/programs /_build/tests /mnt/_build
         sudo chmod a+rwx /_build/libraries /_build/programs /_build/tests
@@ -89,13 +98,13 @@ jobs:
         export CCACHE_DIR="$GITHUB_WORKSPACE/ccache"
         mkdir -p "$CCACHE_DIR"
         df -h
-        make -j 1 -C _build chain_test
-        make -j 1 -C _build cli_test
-        make -j 1 -C _build app_test
-        make -j 1 -C _build es_test
-        make -j 1 -C _build cli_wallet
-        make -j 1 -C _build witness_node
-        make -j 1 -C _build
+        make -j 2 -C _build chain_test
+        make -j 2 -C _build cli_test
+        make -j 2 -C _build app_test
+        make -j 2 -C _build es_test
+        make -j 2 -C _build cli_wallet
+        make -j 2 -C _build witness_node
+        make -j 2 -C _build
         df -h
         du -hs _build/libraries/* _build/programs/* _build/tests/*
         du -hs _build/*
@@ -105,9 +114,8 @@ jobs:
         _build/tests/app_test -l test_suite
         df -h
         rm -rf /tmp/graphene*
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
+        _build/tests/es_test -l test_suite
+        export GRAPHENE_TESTING_ES_URL=http://127.0.0.1:9201/
         _build/tests/es_test -l test_suite
         df -h
         rm -rf /tmp/graphene*

diff --git a/.github/workflows/build-and-test.ubuntu-release.yml b/.github/workflows/build-and-test.ubuntu-release.yml
@@ -8,12 +8,26 @@ jobs:
     name: Build and test in Release mode
     strategy:
       matrix:
-        os: [ ubuntu-18.04, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env ES_JAVA_OPTS="-Xms512m -Xmx512m"
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
+      elasticsearch7:
+        image: elastic/elasticsearch:7.17.8
+        options: >-
+          --env ES_JAVA_OPTS="-Xms512m -Xmx512m"
+          --env discovery.type=single-node
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9201:9200
     steps:
     - name: Install dependencies
       run: |
@@ -45,6 +59,8 @@ jobs:
         submodules: recursive
     - name: Configure
       run: |
+        df -h
+        free
         mkdir -p _build
         pushd _build
         export -n BOOST_ROOT BOOST_INCLUDEDIR BOOST_LIBRARYDIR
@@ -68,14 +84,13 @@ jobs:
       run: |
         export CCACHE_DIR="$GITHUB_WORKSPACE/ccache"
         mkdir -p "$CCACHE_DIR"
-        make -j 1 -C _build
+        make -j 2 -C _build
         df -h
     - name: Unit-Tests
       run: |
         _build/tests/app_test -l test_suite
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
+        _build/tests/es_test -l test_suite
+        export GRAPHENE_TESTING_ES_URL=http://127.0.0.1:9201/
         _build/tests/es_test -l test_suite
         libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
         _build/tests/cli_test -l test_suite

diff --git a/.github/workflows/sonar-scan.yml b/.github/workflows/sonar-scan.yml
@@ -11,9 +11,16 @@ jobs:
         os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env ES_JAVA_OPTS="-Xms512m -Xmx512m"
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
     steps:
     - name: Download and install latest SonarScanner CLI tool
       run: |
@@ -70,11 +77,6 @@ jobs:
         pwd
         df -h .
         free
-        sudo dd if=/dev/zero of=/swapfile bs=1024 count=4M
-        sudo chmod 600 /swapfile
-        sudo mkswap /swapfile
-        sudo swapon /swapfile
-        free
         mkdir -p _build
         sudo mkdir -p /_build/libraries /_build/programs /mnt/_build/tests
         sudo chmod a+rwx /_build/libraries /_build/programs /mnt/_build/tests
@@ -117,7 +119,7 @@ jobs:
         export CCACHE_DIR="$GITHUB_WORKSPACE/ccache"
         mkdir -p "$CCACHE_DIR"
         df -h
-        programs/build_helpers/make_with_sonar bw-output -j 1 -C _build \
+        programs/build_helpers/make_with_sonar bw-output -j 2 -C _build \
           witness_node cli_wallet js_operation_serializer get_dev_key network_mapper \
           app_test chain_test cli_test es_test
         df -h
@@ -167,9 +169,6 @@ jobs:
         df -h
         echo "Cleanup"
         rm -rf /tmp/graphene*
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
         _build/tests/es_test -l test_suite
         df -h
         echo "Cleanup"

diff --git a/docker/default_config.ini b/docker/default_config.ini
@@ -213,13 +213,10 @@ max-ops-per-account = 100
 # ==============================================================================
 
 # Track market history by grouping orders into buckets of equal size measured in seconds specified as a JSON array of numbers
-# bucket-size = [15,60,300,3600,86400]
-bucket-size = [60,300,900,1800,3600,14400,86400]
-# for 1 min, 5 mins, 30 mins, 1h, 4 hs and 1 day. i think this should be the default.
-# https://github.com/bitshares/bitshares-core/issues/465
+bucket-size = [60,300,900,3600,14400,86400,604800]
 
 # How far back in time to track history for each bucket size, measured in the number of buckets (default: 1000)
-history-per-size = 1000
+history-per-size = 1500
 
 # Will only store this amount of matched orders for each market in order history for querying, or those meet the other option, which has more data (default: 1000)
 max-order-his-records-per-market = 1000

diff --git a/libraries/chain/db_management.cpp b/libraries/chain/db_management.cpp
@@ -65,7 +65,7 @@ void database::reindex( fc::path data_dir )
    ilog( "reindexing blockchain" );
    auto start = fc::time_point::now();
    const auto last_block_num = last_block->block_num();
-   uint32_t undo_point = last_block_num < GRAPHENE_MAX_UNDO_HISTORY ? 0 : last_block_num - GRAPHENE_MAX_UNDO_HISTORY;
+   uint32_t undo_point = last_block_num < GRAPHENE_MAX_UNDO_HISTORY ? 0 : (last_block_num - GRAPHENE_MAX_UNDO_HISTORY);
 
    ilog( "Replaying blocks, starting at ${next}...", ("next",head_block_num() + 1) );
    if( head_block_num() >= undo_point )
@@ -88,10 +88,11 @@ void database::reindex( fc::path data_dir )
       if( next_block_num <= last_block_num && blocks.size() < 20 )
       {
          const size_t processed_block_size = _block_id_to_block.blocks_current_position();
-         fc::optional< signed_block > block = _block_id_to_block.fetch_by_number( next_block_num++ );
+         fc::optional< signed_block > block = _block_id_to_block.fetch_by_number( next_block_num );
+         ++next_block_num;
          if( block.valid() )
          {
-            if( block->timestamp >= last_block->timestamp - gpo.parameters.maximum_time_until_expiration )
+            if( block->timestamp >= (last_block->timestamp - gpo.parameters.maximum_time_until_expiration) )
                skip &= (uint32_t)(~skip_transaction_dupe_check);
             blocks.emplace( processed_block_size, std::move(*block), fc::future<void>() );
             std::get<2>(blocks.back()) = precompute_parallel( std::get<1>(blocks.back()), skip );
@@ -104,13 +105,12 @@ void database::reindex( fc::path data_dir )
             {
                fc::optional< block_id_type > last_id = _block_id_to_block.last_id();
                // this can trigger if we attempt to e.g. read a file that has block #2 but no block #1
-               if( !last_id.valid() )
-                  break;
+               // OR
                // we've caught up to the gap
-               if( block_header::num_from_id( *last_id ) <= i )
+               if( !last_id.valid() || block_header::num_from_id( *last_id ) <= i )
                   break;
                _block_id_to_block.remove( *last_id );
-               dropped_count++;
+               ++dropped_count;
             }
             wlog( "Dropped ${n} blocks from after the gap", ("n", dropped_count) );
             next_block_num = last_block_num + 1; // don't load more blocks
@@ -128,8 +128,8 @@ void database::reindex( fc::path data_dir )
             size_t current_pos = std::get<0>(blocks.front());
             if( current_pos > total_block_size )
                total_block_size = current_pos;
-            bysize << std::fixed << std::setprecision(5) << double(current_pos) / total_block_size * 100;
-            bynum << std::fixed << std::setprecision(5) << double(i)*100/last_block_num;
+            bysize << std::fixed << std::setprecision(5) << (100 * double(current_pos) / total_block_size);
+            bynum << std::fixed << std::setprecision(5) << (100 * double(i) / last_block_num);
             ilog(
                "   [by size: ${size}%   ${processed} of ${total}]   [by num: ${num}%   ${i} of ${last}]",
                ("size", bysize.str())
@@ -154,7 +154,7 @@ void database::reindex( fc::path data_dir )
             push_block( block, skip );
          }
          blocks.pop();
-         i++;
+         ++i;
       }
    }
    _undo_db.enable();
@@ -227,17 +227,16 @@ void database::open(
    FC_CAPTURE_LOG_AND_RETHROW( (data_dir) )
 }
 
-void database::close(bool rewind)
+void database::close(bool rewinding)
 {
    if (!_opened)
       return;
-
    // TODO:  Save pending tx's on close()
    clear_pending();
 
    // pop all of the blocks that we can given our undo history, this should
    // throw when there is no more undo history to pop
-   if( rewind )
+   if( rewinding )
    {
       try
       {

diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp
@@ -630,7 +630,8 @@ operation_history_object elasticsearch_plugin::get_operation_by_id( const operat
    }
    )";
 
-   const auto response = my->es->query( my->_options.index_prefix + "*/_doc/_search", query );
+   const auto uri = my->_options.index_prefix + ( my->is_es_version_7_or_above ? "*/_search" : "*/_doc/_search" );
+   const auto response = my->es->query( uri, query );
    variant variant_response = fc::json::from_string(response);
    const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"];
    return fromEStoOperation(source);
@@ -677,7 +678,8 @@ vector<operation_history_object> elasticsearch_plugin::get_account_history(
    if( !my->es->check_status() )
       return result;
 
-   const auto response = my->es->query( my->_options.index_prefix + "*/_doc/_search", query );
+   const auto uri = my->_options.index_prefix + ( my->is_es_version_7_or_above ? "*/_search" : "*/_doc/_search" );
+   const auto response = my->es->query( uri, query );
 
    variant variant_response = fc::json::from_string(response);
 

diff --git a/libraries/plugins/market_history/market_history_plugin.cpp b/libraries/plugins/market_history/market_history_plugin.cpp
@@ -744,12 +744,14 @@ void market_history_plugin::plugin_set_program_options(
    )
 {
    cli.add_options()
-         ("bucket-size", boost::program_options::value<string>()->default_value("[60,300,900,1800,3600,14400,86400]"),
+         ("bucket-size",
+           // 1m, 5m, 15m, 1h, 4h, 1d, 1w
+           boost::program_options::value<string>()->default_value("[60,300,900,3600,14400,86400,604800]"),
            "Track market history by grouping orders into buckets of equal size measured "
            "in seconds specified as a JSON array of numbers")
-         ("history-per-size", boost::program_options::value<uint32_t>()->default_value(1000),
+         ("history-per-size", boost::program_options::value<uint32_t>()->default_value(1500),
            "How far back in time to track history for each bucket size, "
-           "measured in the number of buckets (default: 1000)")
+           "measured in the number of buckets (default: 1500)")
          ("max-order-his-records-per-market", boost::program_options::value<uint32_t>()->default_value(1000),
            "Will only store this amount of matched orders for each market in order history for querying, "
            "or those meet the other option, which has more data (default: 1000). "

diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp
@@ -211,13 +211,14 @@ std::string es_client::get_version() const
 
    fc::variant content = fc::json::from_string( response.content );
    return content["version"]["number"].as_string();
-} FC_CAPTURE_AND_RETHROW() }
+} FC_CAPTURE_LOG_AND_RETHROW( (base_url) ) }
 
 void es_client::check_version_7_or_above( bool& result ) const noexcept
 {
    static const int64_t version_7 = 7;
    try {
       const auto es_version = get_version();
+      ilog( "ES version detected: ${v}", ("v", es_version) );
       auto dot_pos = es_version.find('.');
       result = ( std::stoi(es_version.substr(0,dot_pos)) >= version_7 );
    }

diff --git a/sonar-project.properties b/sonar-project.properties
@@ -2,12 +2,12 @@ sonar.organization=bitshares-on-github
 
 sonar.projectKey=bitshares_bitshares-core
 sonar.projectName=BitShares-Core
-sonar.projectDescription=BitShares Blockchain implementation and command-line interface
+sonar.projectDescription=BitShares Blockchain node and command-line wallet
 sonar.projectVersion=7.0.x
 
 sonar.host.url=https://sonarcloud.io
 
-sonar.links.homepage=https://bitshares.org
+sonar.links.homepage=https://bitshares.github.io
 sonar.links.ci=https://github.com/bitshares/bitshares-core/actions
 sonar.links.issue=https://github.com/bitshares/bitshares-core/issues
 sonar.links.scm=https://github.com/bitshares/bitshares-core/tree/master

diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp
@@ -407,7 +407,6 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
    }
    // load ES or AH, but not both
    if(fixture.current_test_name == "elasticsearch_account_history" ||
-         fixture.current_test_name == "elasticsearch_suite" ||
          fixture.current_test_name == "elasticsearch_history_api") {
       fixture.app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>(true);
 
@@ -429,7 +428,7 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
       fixture.app.register_plugin<graphene::account_history::account_history_plugin>(true);
    }
 
-   if(fixture.current_test_name == "elasticsearch_objects" || fixture.current_test_name == "elasticsearch_suite") {
+   if( fixture.current_test_name == "elasticsearch_objects" ) {
       fixture.app.register_plugin<graphene::es_objects::es_objects_plugin>(true);
 
       fc::set_option( options, "es-objects-elasticsearch-url", GRAPHENE_TESTING_ES_URL );