Skip to content

Commit b750fb2

Browse files
moneromooo-moneromoneromooo-monero
committed
Pruning
The blockchain prunes seven eighths of prunable tx data. This saves about two thirds of the blockchain size, while keeping the node useful as a sync source for an eighth of the blockchain. No other data is currently pruned. There are three ways to prune a blockchain: - run monerod with --prune-blockchain - run "prune_blockchain" in the monerod console - run the monero-blockchain-prune utility The first two will prune in place. Due to how LMDB works, this will not reduce the blockchain size on disk. Instead, it will mark parts of the file as free, so that future data will use that free space, causing the file to not grow until free space grows scarce. The third way will create a second database, a pruned copy of the original one. Since this is a new file, this one will be smaller than the original one. Once the database is pruned, it will stay pruned as it syncs. That is, there is no need to use --prune-blockchain again, etc.
1 parent 4e72384 commit b750fb2

52 files changed

Lines changed: 3292 additions & 429 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎contrib/epee/include/misc_log_ex.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,16 @@
8585
#define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x)
8686
#define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x)
8787

88+
#define IFLOG(level, cat, type, init, x) \
89+
do { \
90+
if (ELPP->vRegistry()->allowed(level, cat)) { \
91+
init; \
92+
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
93+
} \
94+
} while(0)
95+
#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x)
96+
97+
8898
#define LOG_ERROR(x) MERROR(x)
8999
#define LOG_PRINT_L0(x) MWARNING(x)
90100
#define LOG_PRINT_L1(x) MINFO(x)

‎contrib/epee/include/net/abstract_tcp_server2.inl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
295295
CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex);
296296
m_throttle_speed_in.handle_trafic_exact(bytes_transferred);
297297
context.m_current_speed_down = m_throttle_speed_in.get_current_speed();
298+
context.m_max_speed_down = std::max(context.m_max_speed_down, context.m_current_speed_down);
298299
}
299300

300301
{
@@ -497,6 +498,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
497498
CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex);
498499
m_throttle_speed_out.handle_trafic_exact(cb);
499500
context.m_current_speed_up = m_throttle_speed_out.get_current_speed();
501+
context.m_max_speed_up = std::max(context.m_max_speed_up, context.m_current_speed_up);
500502
}
501503

502504
//_info("[sock " << socket_.native_handle() << "] SEND " << cb);

‎contrib/epee/include/net/net_utils_base.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ namespace net_utils
228228
uint64_t m_send_cnt;
229229
double m_current_speed_down;
230230
double m_current_speed_up;
231+
double m_max_speed_down;
232+
double m_max_speed_up;
231233

232234
connection_context_base(boost::uuids::uuid connection_id,
233235
const network_address &remote_address, bool is_income,
@@ -242,7 +244,9 @@ namespace net_utils
242244
m_recv_cnt(recv_cnt),
243245
m_send_cnt(send_cnt),
244246
m_current_speed_down(0),
245-
m_current_speed_up(0)
247+
m_current_speed_up(0),
248+
m_max_speed_down(0),
249+
m_max_speed_up(0)
246250
{}
247251

248252
connection_context_base(): m_connection_id(),
@@ -254,7 +258,9 @@ namespace net_utils
254258
m_recv_cnt(0),
255259
m_send_cnt(0),
256260
m_current_speed_down(0),
257-
m_current_speed_up(0)
261+
m_current_speed_up(0),
262+
m_max_speed_down(0),
263+
m_max_speed_up(0)
258264
{}
259265

260266
connection_context_base& operator=(const connection_context_base& a)

‎contrib/epee/include/string_tools.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,15 @@ POP_WARNINGS
206206
return boost::lexical_cast<std::string>(val);
207207
}
208208
//----------------------------------------------------------------------------
209+
inline std::string to_string_hex(uint32_t val)
210+
{
211+
std::stringstream ss;
212+
ss << std::hex << val;
213+
std::string s;
214+
ss >> s;
215+
return s;
216+
}
217+
//----------------------------------------------------------------------------
209218

210219
inline bool compare_no_case(const std::string& str1, const std::string& str2)
211220
{

‎src/blockchain_db/blockchain_db.cpp‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
267267

268268
for (const auto& h : boost::adaptors::reverse(blk.tx_hashes))
269269
{
270-
txs.push_back(get_tx(h));
270+
cryptonote::transaction tx;
271+
if (!get_tx(h, tx) && !get_pruned_tx(h, tx))
272+
throw DB_ERROR("Failed to get pruned or unpruned transaction from the db");
273+
txs.push_back(std::move(tx));
271274
remove_transaction(h);
272275
}
273276
remove_transaction(get_transaction_hash(blk.miner_tx));
@@ -280,7 +283,7 @@ bool BlockchainDB::is_open() const
280283

281284
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
282285
{
283-
transaction tx = get_tx(tx_hash);
286+
transaction tx = get_pruned_tx(tx_hash);
284287

285288
for (const txin_v& tx_input : tx.vin)
286289
{
@@ -325,6 +328,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co
325328
return true;
326329
}
327330

331+
bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const
332+
{
333+
blobdata bd;
334+
if (!get_pruned_tx_blob(h, bd))
335+
return false;
336+
if (!parse_and_validate_tx_base_from_blob(bd, tx))
337+
throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db");
338+
339+
return true;
340+
}
341+
328342
transaction BlockchainDB::get_tx(const crypto::hash& h) const
329343
{
330344
transaction tx;
@@ -333,6 +347,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const
333347
return tx;
334348
}
335349

350+
transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
351+
{
352+
transaction tx;
353+
if (!get_pruned_tx(h, tx))
354+
throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str());
355+
return tx;
356+
}
357+
336358
void BlockchainDB::reset_stats()
337359
{
338360
num_calls = 0;

‎src/blockchain_db/blockchain_db.h‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,6 +1125,17 @@ class BlockchainDB
11251125
*/
11261126
virtual transaction get_tx(const crypto::hash& h) const;
11271127

1128+
/**
1129+
* @brief fetches the transaction base with the given hash
1130+
*
1131+
* If the transaction does not exist, the subclass should throw TX_DNE.
1132+
*
1133+
* @param h the hash to look for
1134+
*
1135+
* @return the transaction with the given hash
1136+
*/
1137+
virtual transaction get_pruned_tx(const crypto::hash& h) const;
1138+
11281139
/**
11291140
* @brief fetches the transaction with the given hash
11301141
*
@@ -1136,6 +1147,17 @@ class BlockchainDB
11361147
*/
11371148
virtual bool get_tx(const crypto::hash& h, transaction &tx) const;
11381149

1150+
/**
1151+
* @brief fetches the transaction base with the given hash
1152+
*
1153+
* If the transaction does not exist, the subclass should return false.
1154+
*
1155+
* @param h the hash to look for
1156+
*
1157+
* @return true iff the transaction was found
1158+
*/
1159+
virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const;
1160+
11391161
/**
11401162
* @brief fetches the transaction blob with the given hash
11411163
*
@@ -1164,6 +1186,21 @@ class BlockchainDB
11641186
*/
11651187
virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;
11661188

1189+
/**
1190+
* @brief fetches the prunable transaction blob with the given hash
1191+
*
1192+
* The subclass should return the prunable transaction stored which has the given
1193+
* hash.
1194+
*
1195+
* If the transaction does not exist, or if we do not have that prunable data,
1196+
* the subclass should return false.
1197+
*
1198+
* @param h the hash to look for
1199+
*
1200+
* @return true iff the transaction was found and we have its prunable data
1201+
*/
1202+
virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0;
1203+
11671204
/**
11681205
* @brief fetches the prunable transaction hash
11691206
*
@@ -1412,6 +1449,31 @@ class BlockchainDB
14121449
*/
14131450
virtual void prune_outputs(uint64_t amount) = 0;
14141451

1452+
/**
1453+
* @brief get the blockchain pruning seed
1454+
* @return the blockchain pruning seed
1455+
*/
1456+
virtual uint32_t get_blockchain_pruning_seed() const = 0;
1457+
1458+
/**
1459+
* @brief prunes the blockchain
1460+
* @param pruning_seed the seed to use, 0 for default (highly recommended)
1461+
* @return success iff true
1462+
*/
1463+
virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0;
1464+
1465+
/**
1466+
* @brief prunes recent blockchain changes as needed, iff pruning is enabled
1467+
* @return success iff true
1468+
*/
1469+
virtual bool update_pruning() = 0;
1470+
1471+
/**
1472+
* @brief checks pruning was done correctly, iff enabled
1473+
* @return success iff true
1474+
*/
1475+
virtual bool check_pruning() = 0;
1476+
14151477
/**
14161478
* @brief runs a function over all txpool transactions
14171479
*

0 commit comments

Comments
 (0)