Merge pull request #5 from stotiks/path-to-redemption

Path to redemption

Author: stotiks

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "lib/bls-signatures"]
 	path = lib/bls-signatures
 	url = https://github.com/Chia-Network/bls-signatures.git
+[submodule "lib/BLAKE3"]
+	path = lib/BLAKE3
+	url = https://github.com/BLAKE3-team/BLAKE3.git

CMakeLists.txt

@@ -12,49 +12,56 @@ else()
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-unused-function")
 endif()
 
+set(BLAKE3_PATH lib/BLAKE3/c)
+set(FSE_PATH lib/FSE/lib)
+
 include_directories(
 	lib
 	include
-	lib/bls-signatures/src/
+	lib/bls-signatures/src
+	${BLAKE3_PATH}
+	${CMAKE_BINARY_DIR}/_deps/relic-src/include
+	${CMAKE_BINARY_DIR}/_deps/relic-build/include
 )
 
 IF (WIN32)
 set(BLAKE3_SRC
-    lib/b3/blake3.c
-    lib/b3/blake3_portable.c
-    lib/b3/blake3_dispatch.c
-    lib/b3/blake3_avx2.c
-    lib/b3/blake3_avx512.c
-    lib/b3/blake3_sse41.c
+    ${BLAKE3_PATH}/blake3.c
+    ${BLAKE3_PATH}/blake3_portable.c
+    ${BLAKE3_PATH}/blake3_dispatch.c
+    ${BLAKE3_PATH}/blake3_avx2.c
+    ${BLAKE3_PATH}/blake3_avx512.c
+    ${BLAKE3_PATH}/blake3_sse2.c
+    ${BLAKE3_PATH}/blake3_sse41.c
 )
 ELSEIF(OSX_NATIVE_ARCHITECTURE STREQUAL "arm64")
 set(BLAKE3_SRC
-    lib/b3/blake3.c
-    lib/b3/blake3_portable.c
-    lib/b3/blake3_dispatch.c
+    ${BLAKE3_PATH}/blake3.c
+    ${BLAKE3_PATH}/blake3_portable.c
+    ${BLAKE3_PATH}/blake3_dispatch.c
 )
 ELSE()
 set(BLAKE3_SRC
-    lib/b3/blake3.c
-    lib/b3/blake3_portable.c
-    lib/b3/blake3_dispatch.c
-    lib/b3/blake3_avx2_x86-64_unix.S
-    lib/b3/blake3_avx512_x86-64_unix.S
-    lib/b3/blake3_sse41_x86-64_unix.S
+    ${BLAKE3_PATH}/blake3.c
+    ${BLAKE3_PATH}/blake3_portable.c
+    ${BLAKE3_PATH}/blake3_dispatch.c
+    ${BLAKE3_PATH}/blake3_avx2_x86-64_unix.S
+    ${BLAKE3_PATH}/blake3_avx512_x86-64_unix.S
+    ${BLAKE3_PATH}/blake3_sse41_x86-64_unix.S
+    ${BLAKE3_PATH}/blake3_sse2_x86-64_unix.S
 )
 ENDIF()
 
-set(FSE_LIB lib/FSE/lib)
-set(FSE_FILES
-    ${FSE_LIB}/fse_compress.c
-    ${FSE_LIB}/fse_decompress.c
-    ${FSE_LIB}/entropy_common.c
-    ${FSE_LIB}/hist.c
+set(FSE_SRC
+    ${FSE_PATH}/fse_compress.c
+    ${FSE_PATH}/fse_decompress.c
+    ${FSE_PATH}/entropy_common.c
+    ${FSE_PATH}/hist.c
 )
 
 add_library(blake3 STATIC ${BLAKE3_SRC})
 
-add_library(fse STATIC ${FSE_FILES})
+add_library(fse STATIC ${FSE_SRC})
 
 add_library(chia_plotter STATIC
 	lib/chacha8.c
@@ -66,6 +73,7 @@ else()
 	target_link_libraries(chia_plotter blake3 fse Threads::Threads)
 endif()
 
+add_executable(test_copy test/test_copy.cpp)
 add_executable(test_disk_sort test/test_disk_sort.cpp)
 
 add_executable(test_phase_1 test/test_phase_1.cpp)
@@ -82,6 +90,7 @@ if (MSVC)
 	add_library(uint128 STATIC uint128_t/uint128_t.cpp)
 	target_include_directories(uint128 PUBLIC uint128_t)
 
+	target_link_libraries(test_copy chia_plotter)
 	target_link_libraries(test_disk_sort chia_plotter uint128)
 	target_link_libraries(test_phase_1 chia_plotter uint128)
 	target_link_libraries(test_phase_2 chia_plotter uint128)
@@ -94,15 +103,16 @@ if (MSVC)
 
 else()
 
-	target_link_libraries(test_disk_sort chia_plotter)
-	
-	target_link_libraries(test_phase_1 chia_plotter)
-	target_link_libraries(test_phase_2 chia_plotter)
-	target_link_libraries(test_phase_3 chia_plotter)
-	target_link_libraries(test_phase_4 chia_plotter)
-	
-	target_link_libraries(check_phase_1 chia_plotter)
-	
-	target_link_libraries(chia_plot chia_plotter bls)
+target_link_libraries(test_copy chia_plotter)
+target_link_libraries(test_disk_sort chia_plotter)
+
+target_link_libraries(test_phase_1 chia_plotter)
+target_link_libraries(test_phase_2 chia_plotter)
+target_link_libraries(test_phase_3 chia_plotter)
+target_link_libraries(test_phase_4 chia_plotter)
+
+target_link_libraries(check_phase_1 chia_plotter)
+
+target_link_libraries(chia_plot chia_plotter bls)
 
 endif()

README.md

@@ -9,22 +9,29 @@ simply by increasing the number of "cores", ie. threads.
 ## Usage
 
 ```
-chia_plot <pool_key> <farmer_key> [tmp_dir] [tmp_dir2] [num_threads] [log_num_buckets]
-
-For <pool_key> and <farmer_key> see output of `chia keys show`.
-<tmp_dir> needs about 200G space, it will handle about 25% of all writes. (Examples: './', '/mnt/tmp/')
-<tmp_dir2> needs about 110G space and ideally is a RAM drive, it will handle about 75% of all writes.
-If <tmp_dir> is not specified it defaults to current directory.
-If <tmp_dir2> is not specified it defaults to <tmp_dir>.
-[num_threads] defaults to 4, it's recommended to use number of physical cores.
-[log_num_buckets] defaults to 7 (2^7 = 128)
+For <poolkey> and <farmerkey> see output of `chia keys show`.
+<tmpdir> needs about 220 GiB space, it will handle about 25% of all writes. (Examples: './', '/mnt/tmp/')
+<tmpdir2> needs about 110 GiB space and ideally is a RAM drive, it will handle about 75% of all writes.
+
+Usage:
+  chia_plot [OPTION...]
+
+  -n, --count arg      Number of plots to create (default = 1, -1 = infinite)
+  -r, --threads arg    Number of threads (default = 4)
+  -u, --buckets arg    Number of buckets (default = 128)
+  -t, --tmpdir arg     Temporary directory, needs ~220 GiB (default = $PWD)
+  -2, --tmpdir2 arg    Temporary directory 2, needs ~110 GiB [RAM] (default = <tmpdir>)
+  -d, --finaldir arg   Final directory (default = <tmpdir>)
+  -p, --poolkey arg    Pool Public Key (48 bytes)
+  -f, --farmerkey arg  Farmer Public Key (48 bytes)
+      --help           Print help
 ```
 
-Make sure to crank up `<num_threads>` if you have plenty of cores, the default is 4.
+Make sure to crank up `<threads>` if you have plenty of cores, the default is 4.
 Depending on the phase more threads will be launched, the setting is just a multiplier.
 
-RAM usage depends on `<num_threads>` and `<log_num_buckets>`.
-With default `<log_num_buckets>` and 4 threads it's ~2GB total, with 16 threads it's ~6GB total.
+RAM usage depends on `<threads>` and `<buckets>`.
+With default `<buckets>` and 4 threads it's ~2GB total, with 16 threads it's ~6GB total.
 
 ## How to Support
 
@@ -34,67 +41,69 @@ I developed this on my own time, even though I already filled all my HDDs (~50 T
 
 ## Results
 
-On a dual Xeon(R) [email protected] R720 with 256GB RAM and a 3x800GB SATA SSD RAID0, using a 110G tmpfs for `<tmp_dir2>`:
+On a dual Xeon(R) [email protected] R720 with 256GB RAM and a 3x800GB SATA SSD RAID0, using a 110G tmpfs for `<tmpdir2>`:
 
 ```
 Number of Threads: 16
 Number of Sort Buckets: 2^7 (128)
-Working Directory:   ./
-Working Directory 2: ./ram/
-[P1] Table 1 took 21.0467 sec
-[P1] Table 2 took 152.6 sec, found 4295044959 matches
-[P1] Lost 77279 matches due to 32-bit overflow.
-[P1] Table 3 took 181.169 sec, found 4295030463 matches
-[P1] Lost 62514 matches due to 32-bit overflow.
-[P1] Table 4 took 223.303 sec, found 4295044715 matches
-[P1] Lost 76928 matches due to 32-bit overflow.
-[P1] Table 5 took 232.129 sec, found 4294967739 matches
-[P1] Lost 235 matches due to 32-bit overflow.
-[P1] Table 6 took 221.468 sec, found 4294932892 matches
-[P1] Table 7 took 182.597 sec, found 4294838936 matches
-Phase 1 took 1214.37 sec
-[P2] max_table_size = 4295044959
-[P2] Table 7 scan took 16.9198 sec
-[P2] Table 7 rewrite took 44.796 sec, dropped 0 entries (0 %)
-[P2] Table 6 scan took 47.5287 sec
-[P2] Table 6 rewrite took 81.2195 sec, dropped 581301544 entries (13.5346 %)
-[P2] Table 5 scan took 46.6094 sec
-[P2] Table 5 rewrite took 77.9914 sec, dropped 761979000 entries (17.7412 %)
-[P2] Table 4 scan took 52.427 sec
-[P2] Table 4 rewrite took 75.7487 sec, dropped 828872625 entries (19.2983 %)
-[P2] Table 3 scan took 54.0839 sec
-[P2] Table 3 rewrite took 74.9016 sec, dropped 855088153 entries (19.9088 %)
-[P2] Table 2 scan took 49.692 sec
-[P2] Table 2 rewrite took 73.0273 sec, dropped 865610902 entries (20.1537 %)
-Phase 2 took 721.638 sec
+Working Directory:   /mnt/tmp3/chia/tmp/
+Working Directory 2: /mnt/tmp3/chia/tmp/ram/
+[P1] Table 1 took 18.2322 sec
+[P1] Table 2 took 152.806 sec, found 4294955136 matches
+[P1] Table 3 took 181.893 sec, found 4295004058 matches
+[P1] Lost 36189 matches due to 32-bit overflow.
+[P1] Table 4 took 207.631 sec, found 4295076024 matches
+[P1] Lost 108422 matches due to 32-bit overflow.
+[P1] Table 5 took 210.324 sec, found 4295073698 matches
+[P1] Lost 106662 matches due to 32-bit overflow.
+[P1] Table 6 took 205.312 sec, found 4294986463 matches
+[P1] Lost 19157 matches due to 32-bit overflow.
+[P1] Table 7 took 166.982 sec, found 4294910985 matches
+Phase 1 took 1143.2 sec
+[P2] max_table_size = 4295076024
+[P2] Table 7 scan took 16.6775 sec
+[P2] Table 7 rewrite took 40.4863 sec, dropped 0 entries (0 %)
+[P2] Table 6 scan took 47.6553 sec
+[P2] Table 6 rewrite took 66.2653 sec, dropped 581270438 entries (13.5337 %)
+[P2] Table 5 scan took 45.4674 sec
+[P2] Table 5 rewrite took 67.3295 sec, dropped 762045018 entries (17.7423 %)
+[P2] Table 4 scan took 44.4048 sec
+[P2] Table 4 rewrite took 62.1307 sec, dropped 828943971 entries (19.2999 %)
+[P2] Table 3 scan took 45.1361 sec
+[P2] Table 3 rewrite took 61.2604 sec, dropped 855078559 entries (19.9087 %)
+[P2] Table 2 scan took 46.5209 sec
+[P2] Table 2 rewrite took 61.3043 sec, dropped 865523252 entries (20.1521 %)
+Phase 2 took 635.101 sec
 Wrote plot header with 268 bytes
-[P3-1] Table 2 took 76.0894 sec, wrote 3429434057 right entries
-[P3-2] Table 2 took 75.1076 sec, wrote 3429434057 left entries, 3429434057 final
-[P3-1] Table 3 took 78.0162 sec, wrote 3439942310 right entries
-[P3-2] Table 3 took 73.0284 sec, wrote 3439942310 left entries, 3439942310 final
-[P3-1] Table 4 took 133.769 sec, wrote 3466172090 right entries
-[P3-2] Table 4 took 76.1504 sec, wrote 3466172090 left entries, 3466172090 final
-[P3-1] Table 5 took 127.125 sec, wrote 3532988739 right entries
-[P3-2] Table 5 took 77.7182 sec, wrote 3532988739 left entries, 3532988739 final
-[P3-1] Table 6 took 134.779 sec, wrote 3713631348 right entries
-[P3-2] Table 6 took 81.9068 sec, wrote 3713631348 left entries, 3713631348 final
-[P3-1] Table 7 took 69.066 sec, wrote 4294838936 right entries
-[P3-2] Table 7 took 94.0157 sec, wrote 4294838936 left entries, 4294838936 final
-Phase 3 took 1104.11 sec, wrote 21877007480 entries to final plot
+[P3-1] Table 2 took 80.6498 sec, wrote 3429431884 right entries
+[P3-2] Table 2 took 74.787 sec, wrote 3429431884 left entries, 3429431884 final
+[P3-1] Table 3 took 70.8236 sec, wrote 3439925499 right entries
+[P3-2] Table 3 took 77.8787 sec, wrote 3439925499 left entries, 3439925499 final
+[P3-1] Table 4 took 71.894 sec, wrote 3466132053 right entries
+[P3-2] Table 4 took 76.3172 sec, wrote 3466132053 left entries, 3466132053 final
+[P3-1] Table 5 took 72.6806 sec, wrote 3533028680 right entries
+[P3-2] Table 5 took 81.5741 sec, wrote 3533028680 left entries, 3533028680 final
+[P3-1] Table 6 took 77.1653 sec, wrote 3713716025 right entries
+[P3-2] Table 6 took 85.9674 sec, wrote 3713716025 left entries, 3713716025 final
+[P3-1] Table 7 took 75.5146 sec, wrote 4294910985 right entries
+[P3-2] Table 7 took 93.9135 sec, wrote 4294910985 left entries, 4294910985 final
+Phase 3 took 946.026 sec, wrote 21877145126 entries to final plot
 [P4] Starting to write C1 and C3 tables
 [P4] Finished writing C1 and C3 tables
 [P4] Writing C2 table
 [P4] Finished writing C2 table
-Phase 4 took 89.0748 sec, final plot size is 108834390977 bytes
-Total plot creation time was 3129.28 sec
+Phase 4 took 79.6722 sec, final plot size is 108835267927 bytes
+Total plot creation time was 2804.06 sec
 ```
 
 ## How to Verify
 
 To make sure the plots are valid you can use the `ProofOfSpace` tool from `chiapos`:
 
 ```
-ProofOfSpace check -f plot-k32-???.plot [num_iterations]
+git clone https://github.com/Chia-Network/chiapos.git
+cd chiapos && mkdir build && cd build && cmake .. && make -j8
+./ProofOfSpace check -f plot-k32-???.plot [num_iterations]
 ```
 
 ## Future Plans
@@ -113,7 +122,11 @@ keeping most of the load off the CPUs.
 
 ## Install
 
+Ubuntu 20.04
 ```
+sudo apt install  -y libsodium-dev  libgmp3-dev  cmake g++ git
+git clone https://github.com/madMAx43v3r/chia-plotter.git 
+cd  chia-plotter
 git submodule update --init
 ./make_devel.sh
 ```

include/chia/ThreadPool.h

@@ -101,11 +101,16 @@ class ThreadPool : public Processor<T> {
 private:
 	void wrapper(thread_t* state, thread_t* prev, T& input)
 	{
+		uint64_t job = -1;
+		{
+			std::lock_guard<std::mutex> lock(state->mutex);
+			job = state->job;
+		}
 		S out;
 		execute(input, out, state->local);
 		{
 			std::unique_lock<std::mutex> lock(prev->mutex);
-			while(prev->job < state->job) {
+			while(prev->job < job) {
 				prev->signal.wait(lock);
 			}
 		}

include/chia/copy.h

@@ -0,0 +1,70 @@
+/*
+ * copy.h
+ *
+ *  Created on: Jun 8, 2021
+ *      Author: mad
+ */
+
+#ifndef INCLUDE_CHIA_COPY_H_
+#define INCLUDE_CHIA_COPY_H_
+
+#include <chia/stdiox.hpp>
+#include <chia/settings.h>
+
+#include <string>
+#include <vector>
+#include <stdexcept>
+
+#include <cstdio>
+#include <cstdint>
+
+
+inline
+uint64_t copy_file(const std::string& src_path, const std::string& dst_path)
+{
+	FILE* src = FOPEN(src_path.c_str(), "rb");
+	if(!src) {
+		throw std::runtime_error("fopen() failed");
+	}
+	FILE* dst = FOPEN(dst_path.c_str(), "wb");
+	if(!dst) {
+		throw std::runtime_error("fopen() failed");
+	}
+	uint64_t total_bytes = 0;
+	std::vector<uint8_t> buffer(g_read_chunk_size);
+	while(true) {
+		const auto num_bytes = fread(buffer.data(), 1, buffer.size(), src);
+		if(fwrite(buffer.data(), 1, num_bytes, dst) != num_bytes) {
+			throw std::runtime_error("fwrite() failed");
+		}
+		total_bytes += num_bytes;
+		if(num_bytes < buffer.size()) {
+			break;
+		}
+	}
+	if(fclose(dst)) {
+		throw std::runtime_error("fclose() failed");
+	}
+	fclose(src);
+	return total_bytes;
+}
+
+inline
+uint64_t final_copy(const std::string& src_path, const std::string& dst_path)
+{
+	if(src_path == dst_path) {
+		return 0;
+	}
+	const std::string tmp_dst_path = dst_path + ".tmp";
+	uint64_t total_bytes = 0;
+	if(rename(src_path.c_str(), tmp_dst_path.c_str())) {
+		// try manual copy
+		total_bytes = copy_file(src_path, tmp_dst_path);
+	}
+	remove(src_path.c_str());
+	rename(tmp_dst_path.c_str(), dst_path.c_str());
+	return total_bytes;
+}
+
+
+#endif /* INCLUDE_CHIA_COPY_H_ */

include/chia/phase1.h

@@ -25,6 +25,7 @@ namespace phase1 {
 struct input_t {
 	std::array<uint8_t, 32> id = {};
 	std::vector<uint8_t> memo;
+	std::string plot_name;
 };
 
 struct entry_1 {