From 8663f43f40c7a48f5aa547a9c8dd26ba14246ed3 Mon Sep 17 00:00:00 2001 From: moneroexamples Date: Wed, 16 Oct 2019 12:30:18 +0800 Subject: [PATCH 1/5] block and tx caches removed --- ext/vpetrigocaches/cache.hpp | 138 ------- ext/vpetrigocaches/cache_policy.hpp | 77 ---- ext/vpetrigocaches/fifo_cache_policy.hpp | 53 --- ext/vpetrigocaches/lfu_cache_policy.hpp | 76 ---- ext/vpetrigocaches/lru_cache_policy.hpp | 63 --- main.cpp | 9 - src/CmdLineOptions.cpp | 6 - src/page.h | 466 +++-------------------- 8 files changed, 47 insertions(+), 841 deletions(-) delete mode 100644 ext/vpetrigocaches/cache.hpp delete mode 100644 ext/vpetrigocaches/cache_policy.hpp delete mode 100644 ext/vpetrigocaches/fifo_cache_policy.hpp delete mode 100644 ext/vpetrigocaches/lfu_cache_policy.hpp delete mode 100644 ext/vpetrigocaches/lru_cache_policy.hpp diff --git a/ext/vpetrigocaches/cache.hpp b/ext/vpetrigocaches/cache.hpp deleted file mode 100644 index c543ecd..0000000 --- a/ext/vpetrigocaches/cache.hpp +++ /dev/null @@ -1,138 +0,0 @@ -#ifndef CACHE_HPP -#define CACHE_HPP - -#include -#include -#include -#include -#include -#include "cache_policy.hpp" - -namespace caches -{ - -// Base class for caching algorithms -template > -class fixed_sized_cache -{ - public: - - using iterator = typename std::unordered_map::iterator; - - using const_iterator = - typename std::unordered_map::const_iterator; - - using operation_guard = typename std::lock_guard; - - fixed_sized_cache( - size_t max_size, - const Policy& policy = Policy()) - : max_cache_size{max_size}, - cache_policy(policy) - { - if (max_cache_size == 0) - { - max_cache_size = std::numeric_limits::max(); - } - } - - void Put(const Key& key, const Value& value) - { - operation_guard{safe_op}; - auto elem_it = FindElem(key); - - if (elem_it == cache_items_map.end()) - { - // add new element to the cache - if (Size() + 1 > max_cache_size) - { - auto disp_candidate_key = cache_policy.ReplCandidate(); - - Erase(disp_candidate_key); - } - - Insert(key, value); - } - else - { - // update previous value - Update(key, value); - } - } - - bool Contains(const Key& key) - { - operation_guard{safe_op}; - auto elem_it = FindElem(key); - return elem_it != cache_items_map.end(); - } - - const Value& Get(const Key& key) const - { - operation_guard{safe_op}; - auto elem_it = FindElem(key); - - if (elem_it == cache_items_map.end()) - { - throw std::range_error{"No such element in the cache"}; - } - cache_policy.Touch(key); - - return elem_it->second; - } - - const size_t Size() const - { - operation_guard{safe_op}; - - return cache_items_map.size(); - } - - // return a key of a displacement candidate - void Clear() - { - operation_guard{safe_op}; - - cache_policy.Clear(); - cache_items_map.clear(); - } - - protected: - - void Insert(const Key& key, const Value& value) - { - cache_policy.Insert(key); - cache_items_map.emplace(std::make_pair(key, value)); - } - - void Erase(const Key& key) - { - cache_policy.Erase(key); - cache_items_map.erase(key); - } - - void Update(const Key& key, const Value& value) - { - cache_policy.Touch(key); - cache_items_map[key] = value; - } - - const_iterator FindElem(const Key& key) const - { - return cache_items_map.find(key); - } - - -private: - - std::unordered_map cache_items_map; - - mutable Policy cache_policy; - mutable std::mutex safe_op; - - size_t max_cache_size; - -}; -} - -#endif // CACHE_HPP diff --git a/ext/vpetrigocaches/cache_policy.hpp b/ext/vpetrigocaches/cache_policy.hpp deleted file mode 100644 index 19cf815..0000000 --- a/ext/vpetrigocaches/cache_policy.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef CACHE_POLICY_HPP -#define CACHE_POLICY_HPP - -#include - -namespace caches -{ - -template - -class ICachePolicy -{ - public: - - virtual ~ICachePolicy() {} - - // handle element insertion in a cache - virtual void Insert(const Key& key) = 0; - - // handle request to the key-element in a cache - virtual void Touch(const Key& key) = 0; - - // handle element deletion from a cache - virtual void Erase(const Key& key) = 0; - - // return a key of a replacement candidate - virtual const Key& ReplCandidate() const = 0; - - // clear the cache - virtual void Clear() = 0; - -}; - -template -class NoCachePolicy : public ICachePolicy -{ - public: - - NoCachePolicy() = default; - - ~NoCachePolicy() override = default; - - void Insert(const Key& key) override - { - key_storage.emplace(key); - } - - void Touch(const Key& key) override - { - // do not do anything - } - - void Erase(const Key& key) override - { - key_storage.erase(key); - } - - // return a key of a displacement candidate - const Key& ReplCandidate() const override - { - return *key_storage.crbegin(); - } - - // return a key of a displacement candidate - void Clear() override - { - key_storage.clear(); - } - - private: - - std::unordered_set key_storage; -}; - -} // namespace caches - -#endif // CACHE_POLICY_HPP diff --git a/ext/vpetrigocaches/fifo_cache_policy.hpp b/ext/vpetrigocaches/fifo_cache_policy.hpp deleted file mode 100644 index fa17757..0000000 --- a/ext/vpetrigocaches/fifo_cache_policy.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef FIFO_CACHE_POLICY_HPP -#define FIFO_CACHE_POLICY_HPP - -#include -#include "cache_policy.hpp" - -namespace caches -{ - -template -class FIFOCachePolicy : public ICachePolicy -{ - public: - - FIFOCachePolicy() = default; - ~FIFOCachePolicy() = default; - - void Insert(const Key& key) override - { - fifo_queue.emplace_front(key); - } - - // handle request to the key-element in a cache - void Touch(const Key& key) override - { - // nothing to do here in the FIFO strategy - } - - // handle element deletion from a cache - void Erase(const Key& key) override - { - fifo_queue.pop_back(); - } - - // return a key of a replacement candidate - const Key& ReplCandidate() const override - { - return fifo_queue.back(); - } - - // return a key of a displacement candidate - void Clear() override - { - fifo_queue.clear(); - } - -private: - - std::list fifo_queue; -}; -} // namespace caches - -#endif // FIFO_CACHE_POLICY_HPP diff --git a/ext/vpetrigocaches/lfu_cache_policy.hpp b/ext/vpetrigocaches/lfu_cache_policy.hpp deleted file mode 100644 index 0f735d7..0000000 --- a/ext/vpetrigocaches/lfu_cache_policy.hpp +++ /dev/null @@ -1,76 +0,0 @@ -#ifndef LFU_CACHE_POLICY_HPP -#define LFU_CACHE_POLICY_HPP - -#include -#include -#include -#include -#include "cache_policy.hpp" - -namespace caches -{ -template -class LFUCachePolicy : public ICachePolicy -{ - public: - - using lfu_iterator = typename std::multimap::iterator; - - LFUCachePolicy() = default; - - ~LFUCachePolicy() override = default; - - void Insert(const Key& key) override - { - constexpr std::size_t INIT_VAL = 1; - - // all new value initialized with the frequency 1 - lfu_storage[key] = frequency_storage.emplace_hint( - frequency_storage.cbegin(), INIT_VAL, key); - } - - void Touch(const Key& key) override - { - // get the previous frequency value of a key - auto elem_for_update = lfu_storage[key]; - - auto updated_elem = std::make_pair( - elem_for_update->first + 1, elem_for_update->second); - - // update the previous value - frequency_storage.erase(elem_for_update); - - lfu_storage[key] = frequency_storage.emplace_hint( - frequency_storage.cend(), std::move(updated_elem)); - } - - void Erase(const Key& key) override - { - frequency_storage.erase(lfu_storage[key]); - lfu_storage.erase(key); - } - - const Key& ReplCandidate() const override - { - // at the beginning of the frequency_storage we have the - // least frequency used value - return frequency_storage.cbegin()->second; - } - - // return a key of a displacement candidate - void Clear() override - { - frequency_storage.clear(); - lfu_storage.clear(); - } - - - private: - - std::multimap frequency_storage; - - std::unordered_map lfu_storage; -}; -} // namespace caches - -#endif // LFU_CACHE_POLICY_HPP diff --git a/ext/vpetrigocaches/lru_cache_policy.hpp b/ext/vpetrigocaches/lru_cache_policy.hpp deleted file mode 100644 index 5dcdac0..0000000 --- a/ext/vpetrigocaches/lru_cache_policy.hpp +++ /dev/null @@ -1,63 +0,0 @@ -#ifndef LRU_CACHE_POLICY_HPP -#define LRU_CACHE_POLICY_HPP - -#include -#include -#include "cache_policy.hpp" - -namespace caches -{ -template -class LRUCachePolicy : public ICachePolicy -{ - - public: - - using lru_iterator = typename std::list::const_iterator; - - LRUCachePolicy() = default; - ~LRUCachePolicy() = default; - - void Insert(const Key& key) override - { - lru_queue.emplace_front(key); - key_finder[key] = lru_queue.cbegin(); - } - - void Touch(const Key& key) override - { - // move the touched element at the beginning of the lru_queue - lru_queue.splice(lru_queue.cbegin(), lru_queue, key_finder[key]); - } - - void Erase(const Key& key) override - { - // remove the least recently used element - key_finder.erase(lru_queue.back()); - - lru_queue.pop_back(); - } - - // return a key of a displacement candidate - const Key& ReplCandidate() const override - { - return lru_queue.back(); - } - - // return a key of a displacement candidate - void Clear() override - { - lru_queue.clear(); - key_finder.clear(); - } - - private: - - std::list lru_queue; - - std::unordered_map key_finder; -}; - -} // namespace caches - -#endif // LRU_CACHE_POLICY_HPP diff --git a/main.cpp b/main.cpp index 7f9e911..d90fcff 100644 --- a/main.cpp +++ b/main.cpp @@ -73,10 +73,7 @@ main(int ac, const char* av[]) auto enable_mixin_details_opt = opts.get_option("enable-mixin-details"); auto enable_json_api_opt = opts.get_option("enable-json-api"); auto enable_as_hex_opt = opts.get_option("enable-as-hex"); - auto enable_tx_cache_opt = opts.get_option("enable-tx-cache"); auto concurrency_opt = opts.get_option("concurrency"); - auto enable_block_cache_opt = opts.get_option("enable-block-cache"); - auto show_cache_times_opt = opts.get_option("show-cache-times"); auto enable_emission_monitor_opt = opts.get_option("enable-emission-monitor"); @@ -101,10 +98,7 @@ main(int ac, const char* av[]) bool enable_mixin_details {*enable_mixin_details_opt}; bool enable_json_api {*enable_json_api_opt}; bool enable_as_hex {*enable_as_hex_opt}; - bool enable_tx_cache {*enable_tx_cache_opt}; - bool enable_block_cache {*enable_block_cache_opt}; bool enable_emission_monitor {*enable_emission_monitor_opt}; - bool show_cache_times {*show_cache_times_opt}; // set monero log output level @@ -279,9 +273,6 @@ main(int ac, const char* av[]) enable_output_key_checker, enable_autorefresh_option, enable_mixin_details, - enable_tx_cache, - enable_block_cache, - show_cache_times, no_blocks_on_index, mempool_info_timeout, *testnet_url, diff --git a/src/CmdLineOptions.cpp b/src/CmdLineOptions.cpp index e52e8af..d8bad20 100644 --- a/src/CmdLineOptions.cpp +++ b/src/CmdLineOptions.cpp @@ -37,12 +37,6 @@ namespace xmreg "enable outputs key file checker") ("enable-json-api", value()->default_value(false)->implicit_value(true), "enable JSON REST api") - ("enable-tx-cache", value()->default_value(false)->implicit_value(true), - "enable caching of transaction details") - ("show-cache-times", value()->default_value(false)->implicit_value(true), - "show times of getting data from cache vs no cache") - ("enable-block-cache", value()->default_value(false)->implicit_value(true), - "enable caching of block details") ("enable-as-hex", value()->default_value(false)->implicit_value(true), "enable links to provide hex represtations of a tx and a block") ("enable-autorefresh-option", value()->default_value(false)->implicit_value(true), diff --git a/src/page.h b/src/page.h index 9cfbf5d..b4bbfb1 100644 --- a/src/page.h +++ b/src/page.h @@ -31,9 +31,6 @@ #include "../ext/json.hpp" -#include "../ext/vpetrigocaches/cache.hpp" -#include "../ext/vpetrigocaches/lru_cache_policy.hpp" -#include "../ext/vpetrigocaches/fifo_cache_policy.hpp" #include "../ext/mstch/src/visitor/render_node.hpp" extern "C" uint64_t me_rx_seedheight(const uint64_t height); @@ -85,31 +82,6 @@ extern __thread randomx_vm *rx_vm; MAKE_ONIONEXPLORER_RPC_VERSION(ONIONEXPLORER_RPC_VERSION_MAJOR, ONIONEXPLORER_RPC_VERSION_MINOR) -// basic info about tx to be stored in cashe. -// we need to store block_no and timestamp, -// as this time and number of confirmation needs -// to be updated between requests. Just cant -// get it from cash, as it will be old very soon -struct tx_info_cache -{ - uint64_t block_no; - uint64_t timestamp; - mstch::map tx_map; - - // custom key for use in cache. - // cache uses unordeded map for keys - struct key - { - crypto::hash tx_hash; - bool detailed; - - bool operator==(const key &other) const - { - return (tx_hash == other.tx_hash && detailed == other.detailed); - } - }; -}; - // helper to ignore any number of template parametrs template using VoidT = void; @@ -147,22 +119,6 @@ struct OutputIndicesReturnVectOfVectT< >>: std::true_type {}; -// indect overload of hash for tx_info_cache::key -namespace std -{ - template<> - struct hash - { - size_t operator()(const tx_info_cache::key& k) const - { - size_t const h1 ( std::hash{}(k.tx_hash) ); - size_t const h2 ( std::hash{}(k.detailed) ); - return h1 ^ (h2 << 1); - }; - }; -} - - /** * visitor to produce json representations of @@ -521,15 +477,10 @@ bool enable_pusher; bool enable_key_image_checker; bool enable_output_key_checker; bool enable_mixins_details; -bool enable_tx_cache; -bool enable_block_cache; bool enable_as_hex; -bool show_cache_times; - bool enable_autorefresh_option; - uint64_t no_of_mempool_tx_of_frontpage; uint64_t no_blocks_on_index; uint64_t mempool_info_timeout; @@ -548,24 +499,6 @@ string js_html_files_all_in_one; // read operation in OS map template_file; - -// alias for easy class typing -template -using lru_cache_t = caches::fixed_sized_cache>; - - -// alias for easy class typing -template -using fifo_cache_t = caches::fixed_sized_cache>; - -// cache of txs_map of txs in blocks. this is useful for -// index2 page, so that we dont parse txs in each block -// for each request. -fifo_cache_t>> block_tx_cache; - -lru_cache_t tx_context_cache; - - public: page(MicroCore* _mcore, @@ -578,9 +511,6 @@ page(MicroCore* _mcore, bool _enable_output_key_checker, bool _enable_autorefresh_option, bool _enable_mixins_details, - bool _enable_tx_cache, - bool _enable_block_cache, - bool _show_cache_times, uint64_t _no_blocks_on_index, uint64_t _mempool_info_timeout, string _testnet_url, @@ -597,16 +527,11 @@ page(MicroCore* _mcore, enable_output_key_checker {_enable_output_key_checker}, enable_autorefresh_option {_enable_autorefresh_option}, enable_mixins_details {_enable_mixins_details}, - enable_tx_cache {_enable_tx_cache}, - enable_block_cache {_enable_block_cache}, - show_cache_times {_show_cache_times}, no_blocks_on_index {_no_blocks_on_index}, mempool_info_timeout {_mempool_info_timeout}, testnet_url {_testnet_url}, stagenet_url {_stagenet_url}, - mainnet_url {_mainnet_url}, - block_tx_cache(200), - tx_context_cache(1000) + mainnet_url {_mainnet_url} { mainnet = nettype == cryptonote::network_type::MAINNET; testnet = nettype == cryptonote::network_type::TESTNET; @@ -705,8 +630,7 @@ index2(uint64_t page_no = 0, bool refresh_page = false) {"enable_pusher" , enable_pusher}, {"enable_key_image_checker" , enable_key_image_checker}, {"enable_output_key_checker", enable_output_key_checker}, - {"enable_autorefresh_option", enable_autorefresh_option}, - {"show_cache_times" , show_cache_times} + {"enable_autorefresh_option", enable_autorefresh_option} }; context.emplace("txs", mstch::array()); // will keep tx to show @@ -724,12 +648,6 @@ index2(uint64_t page_no = 0, bool refresh_page = false) vector blk_sizes; - // measure time of cache based execution, and non-cached execution - double duration_cached {0.0}; - double duration_non_cached {0.0}; - uint64_t cache_hits {0}; - uint64_t cache_misses {0}; - // loop index int64_t i = end_height; @@ -764,193 +682,68 @@ index2(uint64_t page_no = 0, bool refresh_page = false) context["age_format"] = age.second; + // start measure time here + auto start = std::chrono::steady_clock::now(); - if (enable_block_cache && block_tx_cache.Contains(i)) - { - // get txs info in the ith block from - // our cache - - // start measure time here - auto start = std::chrono::steady_clock::now(); - - const vector>& txd_pairs - = block_tx_cache.Get(i); - - // copy tx maps from txs_maps_tmp into txs array, - // that will go to templates - for (const pair& txd_pair: txd_pairs) - { - // we need to check if the given transaction is still - // in the same block as when it was cached. it is possible - // the block got orphaned, and this tx is in mempool - // or different block, and what we have in cache - // is thus wrong - - // but we do this only for first top blocks. no sense - // doing it for all blocks - - bool is_tx_still_in_block_as_expected {true}; - - if (i + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > height) - { - const crypto::hash& tx_hash = txd_pair.first; - - if (core_storage->have_tx(tx_hash)) - { - try - { - uint64_t tx_height_in_blockchain = - core_storage->get_db().get_tx_block_height(tx_hash); - - // check if height of the given tx that we have in cache, - // denoted by i, is same as what is acctually stored - // in blockchain - if (tx_height_in_blockchain == i) - { - is_tx_still_in_block_as_expected = true; - } - else - { - // if no tx in the given block, just stop - // any futher search. no need. we are going - // to ditch the cache, in a monent - is_tx_still_in_block_as_expected = false; - break; - } - } - catch (const TX_DNE& e) - { - cerr << "Tx from cache" << pod_to_hex(tx_hash) - << " is no longer in the blockchain " - << endl; - - is_tx_still_in_block_as_expected = false; - break; - } - } - else - { - is_tx_still_in_block_as_expected = false; - break; - } - - } // if (i + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > height) - - - if (!is_tx_still_in_block_as_expected) - { - // if some tx in cache is not in blockchain - // where it should be, its probably better to - // ditch entire cache, as redo it below. - - block_tx_cache.Clear(); - txs.clear(); - i = end_height; - continue; // reado the main loop - } - - // if we got to here, it means that everything went fine - // and no unexpeced things happended. - mstch::map txd_map = boost::get(txd_pair.second); - - // now we need to update age of txs from cashe - if (!boost::get(txd_map["age"]).empty()) - { - txd_map["age"] = age.first; - } - - txs.push_back(txd_map); - - } // for (const pair& txd_pair: txd_pairs) - - auto duration = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); - - // cout << "block_tx_json_cache from cache" << endl; - - duration_cached += duration.count(); + // get all transactions in the block found + // initialize the first list with transaction for solving + // the block i.e. coinbase. + vector blk_txs {blk.miner_tx}; + vector missed_txs; - ++cache_hits; - } - else + if (!core_storage->get_transactions(blk.tx_hashes, blk_txs, missed_txs)) { - // this is new block. not in cashe. - // need to process its txs and add to cache - - // start measure time here - auto start = std::chrono::steady_clock::now(); - - // get all transactions in the block found - // initialize the first list with transaction for solving - // the block i.e. coinbase. - vector blk_txs {blk.miner_tx}; - vector missed_txs; - - if (!core_storage->get_transactions(blk.tx_hashes, blk_txs, missed_txs)) - { - cerr << "Cant get transactions in block: " << i << endl; - --i; - continue; - } - - uint64_t tx_i {0}; - - // this vector will go into block_tx cache - // tx_hash , txd_map - vector> txd_pairs; - - for(auto it = blk_txs.begin(); it != blk_txs.end(); ++it) - { - const cryptonote::transaction& tx = *it; - - const tx_details& txd = get_tx_details(tx, false, i, height); + cerr << "Cant get transactions in block: " << i << endl; + --i; + continue; + } - mstch::map txd_map = txd.get_mstch_map(); + uint64_t tx_i {0}; - //add age to the txd mstch map - txd_map.insert({"height" , i}); - txd_map.insert({"blk_hash" , blk_hash_str}); - txd_map.insert({"age" , age.first}); - txd_map.insert({"is_ringct" , (tx.version > 1)}); - txd_map.insert({"rct_type" , tx.rct_signatures.type}); - txd_map.insert({"blk_size" , blk_size_str}); + // this vector will go into block_tx cache + // tx_hash , txd_map + vector> txd_pairs; + for(auto it = blk_txs.begin(); it != blk_txs.end(); ++it) + { + const cryptonote::transaction& tx = *it; - // do not show block info for other than first tx in a block - if (tx_i > 0) - { - txd_map["height"] = string(""); - txd_map["age"] = string(""); - txd_map["blk_size"] = string(""); - } + const tx_details& txd = get_tx_details(tx, false, i, height); - txd_pairs.emplace_back(txd.hash, txd_map); + mstch::map txd_map = txd.get_mstch_map(); - ++tx_i; + //add age to the txd mstch map + txd_map.insert({"height" , i}); + txd_map.insert({"blk_hash" , blk_hash_str}); + txd_map.insert({"age" , age.first}); + txd_map.insert({"is_ringct" , (tx.version > 1)}); + txd_map.insert({"rct_type" , tx.rct_signatures.type}); + txd_map.insert({"blk_size" , blk_size_str}); - } // for(list::reverse_iterator rit = blk_txs.rbegin(); - // copy tx maps from txs_maps_tmp into txs array, - // that will go to templates - for (const pair& txd_pair: txd_pairs) + // do not show block info for other than first tx in a block + if (tx_i > 0) { - txs.push_back(boost::get(txd_pair.second)); + txd_map["height"] = string(""); + txd_map["age"] = string(""); + txd_map["blk_size"] = string(""); } - auto duration = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); + txd_pairs.emplace_back(txd.hash, txd_map); - duration_non_cached += duration.count(); + ++tx_i; - ++cache_misses; + } // for(list::reverse_iterator rit = blk_txs.rbegin(); - if (enable_block_cache) - { - // save in block_tx cache - block_tx_cache.Put(i, txd_pairs); - } + // copy tx maps from txs_maps_tmp into txs array, + // that will go to templates + for (const pair& txd_pair: txd_pairs) + { + txs.push_back(boost::get(txd_pair.second)); + } - } // else if (block_tx_json_cache.Contains(i)) + auto duration = std::chrono::duration_cast + (std::chrono::steady_clock::now() - start); --i; // go to next block number @@ -959,21 +752,6 @@ index2(uint64_t page_no = 0, bool refresh_page = false) // calculate median size of the blocks shown //double blk_size_median = xmreg::calc_median(blk_sizes.begin(), blk_sizes.end()); - // save computational times for disply in the frontend - - context["construction_time_cached"] = fmt::format( - "{:0.4f}", duration_cached/1.0e6); - - context["construction_time_non_cached"] = fmt::format( - "{:0.4f}", duration_non_cached/1.0e6); - - context["construction_time_total"] = fmt::format( - "{:0.4f}", (duration_non_cached+duration_cached)/1.0e6); - - context["cache_hits"] = cache_hits; - context["cache_misses"] = cache_misses; - - // get current network info from MemoryStatus thread. MempoolStatus::network_info current_network_info = MempoolStatus::current_network_info; @@ -1101,7 +879,6 @@ mempool(bool add_header_and_footer = false, uint64_t no_of_mempool_tx = 25) // initalise page tempate map with basic info about mempool mstch::map context { {"mempool_size" , static_cast(total_no_of_mempool_tx)}, // total no of mempool txs - {"show_cache_times" , show_cache_times}, {"mempool_refresh_time" , MempoolStatus::mempool_refresh_time} }; @@ -1110,11 +887,6 @@ mempool(bool add_header_and_footer = false, uint64_t no_of_mempool_tx = 25) // get reference to blocks template map to be field below mstch::array& txs = boost::get(context["mempooltxs"]); - double duration_cached {0.0}; - double duration_non_cached {0.0}; - uint64_t cache_hits {0}; - uint64_t cache_misses {0}; - uint64_t local_copy_server_timestamp = server_timestamp; // for each transaction in the memory pool @@ -1145,7 +917,6 @@ mempool(bool add_header_and_footer = false, uint64_t no_of_mempool_tx = 25) delta_time[3], delta_time[4]); } - // cout << "block_tx_json_cache from cache" << endl; // set output page template map txs.push_back(mstch::map { @@ -1583,149 +1354,8 @@ show_tx(string tx_hash_str, uint16_t with_ring_signatures = 0, bool refresh_page mstch::map tx_context; - if (enable_tx_cache && tx_context_cache.Contains({tx_hash, static_cast(with_ring_signatures)})) - { - // with_ring_signatures == 0 means that cache is not used - // when obtaining detailed information about tx is requested. - - // we are going to measure time for the construction of the - // tx context from cashe. just for fun, to see if cache is any faster. - auto start = std::chrono::steady_clock::now(); - - const tx_info_cache& tx_info_cashed - = tx_context_cache.Get({tx_hash, static_cast(with_ring_signatures)}); - - tx_context = tx_info_cashed.tx_map; - - //cout << "get tx from cash: " << tx_hash_str <(tx_context["tx_blk_height"]) <(tx_context["blk_timestamp_uint"]) <(tx_context["tx_blk_height"]); - uint64_t blk_timestamp_uint = boost::get(tx_context["blk_timestamp_uint"]); - - if (tx_blk_height > 0) - { - // seems to be in blockchain. off course it could have been orphaned - // so double check if its for sure in blockchain - - if (core_storage->have_tx(tx_hash)) - { - // ok, it is still in blockchain - // update its age and number of confirmations - - pair age - = get_age(std::time(nullptr), - blk_timestamp_uint, - FULL_AGE_FORMAT); - - tx_context["delta_time"] = age.first; - - uint64_t bc_height = core_storage->get_current_blockchain_height(); - - tx_context["confirmations"] = bc_height - (tx_blk_height - 1); - - // marke it as from cashe. useful if we want to show - // info about cashed/not cashed in frontend. - tx_context["from_cache"] = true; - - auto duration = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); - - tx_context["construction_time"] = fmt::format( - "{:0.4f}", static_cast(duration.count())/1.0e6); - - // normally we should update this into in the cache. - // but since we make this check all the time, - // we can skip updating cashed version - - } // if (core_storage->have_tx(tx_hash)) - else - { - // its not in blockchain, but it was there when we cashed it. - // so we update it in cash, as it should be back in mempool - - tx_context = construct_tx_context(tx, static_cast(with_ring_signatures)); - - tx_context_cache.Put( - {tx_hash, static_cast(with_ring_signatures)}, - tx_info_cache { - boost::get(tx_context["tx_blk_height"]), - boost::get(tx_context["blk_timestamp_uint"]), - tx_context} - ); - } - } // if (tx_blk_height > 0) - else - { - // the tx was cashed when in mempool. - // since then, it might have been included in some block. - // so we check it. - - if (core_storage->have_tx(tx_hash)) - { - // checking if in blockchain already - // it was before in mempool, but now maybe already in blockchain - - tx_context = construct_tx_context(tx, static_cast(with_ring_signatures)); - - tx_context_cache.Put( - {tx_hash, static_cast(with_ring_signatures)}, - tx_info_cache { - boost::get(tx_context["tx_blk_height"]), - boost::get(tx_context["blk_timestamp_uint"]), - tx_context}); - - - } // if (core_storage->have_tx(tx_hash)) - else - { - // still seems to be in mempool only. - // so just get its time duration, as its read only - // from cache - - tx_context["from_cache"] = true; - - auto duration = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); - - tx_context["construction_time"] = fmt::format( - "{:0.4f}", static_cast(duration.count())/1.0e6); - - } - - } // else if (tx_blk_height > 0) - - } // if (tx_context_cache.Contains(tx_hash)) - else - { - - // we are going to measure time for the construction of the - // tx context. just for fun, to see if cache is any faster. - auto start = std::chrono::steady_clock::now(); - - tx_context = construct_tx_context(tx, static_cast(with_ring_signatures)); - - auto duration = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); - - if (enable_tx_cache) - { - tx_context_cache.Put( - {tx_hash, static_cast(with_ring_signatures)}, - tx_info_cache { - boost::get(tx_context["tx_blk_height"]), - boost::get(tx_context["blk_timestamp_uint"]), - tx_context}); - } - - tx_context["construction_time"] = fmt::format( - "{:0.4f}", static_cast(duration.count())/1.0e6); - - } // else if (tx_context_cache.Contains(tx_hash)) + tx_context = construct_tx_context(tx, static_cast(with_ring_signatures)); tx_context["show_more_details_link"] = show_more_details_link; @@ -1737,7 +1367,6 @@ show_tx(string tx_hash_str, uint16_t with_ring_signatures = 0, bool refresh_page mstch::map context { {"testnet" , this->testnet}, {"stagenet" , this->stagenet}, - {"show_cache_times" , show_cache_times}, {"txs" , mstch::array{}}, {"refresh" , refresh_page}, {"tx_hash" , tx_hash_str} @@ -6447,7 +6076,6 @@ construct_tx_context(transaction tx, uint16_t with_ring_signatures = 0) {"error_msg" , string("")}, {"have_raw_tx" , false}, {"show_more_details_link", true}, - {"from_cache" , false}, {"construction_time" , string {}}, }; From ad91f74acd12a6f33baf360cfdee700e7b6e3793 Mon Sep 17 00:00:00 2001 From: moneroexamples Date: Wed, 16 Oct 2019 15:51:00 +0800 Subject: [PATCH 2/5] add try catch to MicroCore::get_tx --- src/MicroCore.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/MicroCore.cpp b/src/MicroCore.cpp index 6d46db5..be2a723 100644 --- a/src/MicroCore.cpp +++ b/src/MicroCore.cpp @@ -137,13 +137,21 @@ MicroCore::get_tx(const crypto::hash& tx_hash, transaction& tx) if (m_blockchain_storage.have_tx(tx_hash)) { // get transaction with given hash - tx = m_blockchain_storage.get_db().get_tx(tx_hash); + try + { + tx = m_blockchain_storage.get_db().get_tx(tx_hash); + } + catch (TX_DNE const& e) + { + cerr << "MicroCore::get_tx: " << e.what() << endl; + return false; + } } else { cerr << "MicroCore::get_tx tx does not exist in blockchain: " << tx_hash << endl; return false; - } + } return true; From 143018b5aa87e936203aef04f24e670142210542 Mon Sep 17 00:00:00 2001 From: moneroexamples Date: Sat, 19 Oct 2019 08:20:00 +0800 Subject: [PATCH 3/5] remove caches from html template --- main.cpp | 1 + src/page.h | 1 - src/templates/index2.html | 11 ----------- src/templates/partials/tx_details.html | 14 -------------- 4 files changed, 1 insertion(+), 26 deletions(-) diff --git a/main.cpp b/main.cpp index d90fcff..0b1815e 100644 --- a/main.cpp +++ b/main.cpp @@ -41,6 +41,7 @@ struct jsonresponse: public crow::response int main(int ac, const char* av[]) { + // get command line options xmreg::CmdLineOptions opts {ac, av}; diff --git a/src/page.h b/src/page.h index b4bbfb1..c74c8cb 100644 --- a/src/page.h +++ b/src/page.h @@ -700,7 +700,6 @@ index2(uint64_t page_no = 0, bool refresh_page = false) uint64_t tx_i {0}; - // this vector will go into block_tx cache // tx_hash , txd_map vector> txd_pairs; diff --git a/src/templates/index2.html b/src/templates/index2.html index 8e6f96c..7c39a2c 100644 --- a/src/templates/index2.html +++ b/src/templates/index2.html @@ -111,14 +111,3 @@ - -{{#show_cache_times}} -
-
- Tx details construction time: {{construction_time_total}} s -
- includes {{construction_time_cached}} s from block cache ({{cache_hits}} hits) - and {{construction_time_non_cached}} s from non cache ({{cache_misses}} misses) -
-
-{{/show_cache_times}} diff --git a/src/templates/partials/tx_details.html b/src/templates/partials/tx_details.html index 5701842..7da9d24 100644 --- a/src/templates/partials/tx_details.html +++ b/src/templates/partials/tx_details.html @@ -300,18 +300,4 @@ {{/with_ring_signatures}} {{/have_raw_tx}} - -{{#show_cache_times}} -
- {{#construction_time}} -
- Tx details construction time: {{construction_time}} s - {{#from_cache}} -
Tx read from the tx cache - {{/from_cache}} -
- {{/construction_time}} -
-{{/show_cache_times}} - From afca5e9dd6ba39650d14d2922c76cfe499e0bb29 Mon Sep 17 00:00:00 2001 From: moneroexamples Date: Sun, 20 Oct 2019 06:48:22 +0800 Subject: [PATCH 4/5] fix coinbase txs --- src/MicroCore.cpp | 12 +++++++++++- src/page.h | 15 +++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/MicroCore.cpp b/src/MicroCore.cpp index be2a723..2c38204 100644 --- a/src/MicroCore.cpp +++ b/src/MicroCore.cpp @@ -143,7 +143,17 @@ MicroCore::get_tx(const crypto::hash& tx_hash, transaction& tx) } catch (TX_DNE const& e) { - cerr << "MicroCore::get_tx: " << e.what() << endl; + try + { + // coinbase txs are not considered pruned + tx = m_blockchain_storage.get_db().get_pruned_tx(tx_hash); + return true; + } + catch (TX_DNE const& e) + { + cerr << "MicroCore::get_tx: " << e.what() << endl; + } + return false; } } diff --git a/src/page.h b/src/page.h index c74c8cb..4fc4521 100644 --- a/src/page.h +++ b/src/page.h @@ -4506,11 +4506,6 @@ json_transaction(string tx_hash_str) no_confirmations = txd.no_confirmations; } - // get tx from tx fetched. can be use to double check - // if what we return in the json response agrees with - // what tx_hash was requested - string tx_hash_str_again = pod_to_hex(get_transaction_hash(tx)); - // get basic tx info j_data = get_tx_json(tx, txd); @@ -6490,7 +6485,15 @@ get_tx_details(const transaction& tx, tx_details txd; // get tx hash - txd.hash = get_transaction_hash(tx); + + if (!tx.pruned) + { + txd.hash = get_transaction_hash(tx); + } + else + { + txd.hash = get_pruned_transaction_hash(tx, tx.prunable_hash); + } // get tx public key from extra // this check if there are two public keys From 21ce4b12d896b48ef9df81d7ab9b0bc8e8417799 Mon Sep 17 00:00:00 2001 From: moneroexamples Date: Tue, 22 Oct 2019 09:58:31 +0800 Subject: [PATCH 5/5] fix: wrong difficult on the front page https://github.com/moneroexamples/onion-monero-blockchain-explorer/issues/195 --- src/page.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/page.h b/src/page.h index 4fc4521..75adcf5 100644 --- a/src/page.h +++ b/src/page.h @@ -757,9 +757,14 @@ index2(uint64_t page_no = 0, bool refresh_page = false) // perapre network info mstch::map for the front page string hash_rate; + double hr_d; char metric_prefix; - cryptonote::difficulty_type hr = make_difficulty(current_network_info.hash_rate, current_network_info.hash_rate_top64); + + cryptonote::difficulty_type hr = make_difficulty( + current_network_info.hash_rate, + current_network_info.hash_rate_top64); + get_metric_prefix(hr, hr_d, metric_prefix); if (metric_prefix != 0) @@ -778,7 +783,7 @@ index2(uint64_t page_no = 0, bool refresh_page = false) } context["network_info"] = mstch::map { - {"difficulty" , make_difficulty(current_network_info.difficulty, current_network_info.difficulty_top64).str()}, + {"difficulty" , current_network_info.difficulty}, {"hash_rate" , hash_rate}, {"fee_per_kb" , print_money(current_network_info.fee_per_kb)}, {"alt_blocks_no" , current_network_info.alt_blocks_count},