mirror of
https://github.com/KingDuckZ/dindexer.git
synced 2025-02-19 12:04:54 +00:00
Re-add some progress feedback.
It's not multithreaded this time o.O
This commit is contained in:
parent
c28398a1b3
commit
43c8024b0c
4 changed files with 67 additions and 9 deletions
|
|
@ -19,9 +19,11 @@
|
|||
#define idC7CC55298AC049EAA80604D6C7FD081D
|
||||
|
||||
#include "dindexer-machinery/scantask/leanbase.hpp"
|
||||
#include "dindexer-machinery/tiger.hpp"
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
#include <boost/utility/string_ref.hpp>
|
||||
#include <cstdint>
|
||||
|
||||
namespace mchlib {
|
||||
struct FileRecordData;
|
||||
|
|
@ -30,15 +32,19 @@ namespace mchlib {
|
|||
class Hashing : public LeanBase<std::vector<FileRecordData>> {
|
||||
public:
|
||||
typedef LeanBase<std::vector<FileRecordData>> FileTreeBase;
|
||||
typedef std::function<void(const boost::string_ref, uint64_t, uint64_t, uint32_t)> ProgressCallback;
|
||||
|
||||
Hashing ( std::shared_ptr<FileTreeBase> parFileTree, bool parIgnoreErrors );
|
||||
virtual ~Hashing ( void ) noexcept;
|
||||
|
||||
void set_progress_callback ( ProgressCallback parFunc );
|
||||
|
||||
private:
|
||||
virtual void on_data_fill ( void ) override;
|
||||
virtual std::vector<FileRecordData>& on_data_get ( void ) override;
|
||||
|
||||
std::shared_ptr<FileTreeBase> m_file_tree_task;
|
||||
ProgressCallback m_progress_callback;
|
||||
bool m_ignore_errors;
|
||||
};
|
||||
} //namespace scantask
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
#include "dindexer-machinery/scantask/hashing.hpp"
|
||||
#include "dindexer-machinery/recorddata.hpp"
|
||||
#include "dindexer-machinery/set_listing.hpp"
|
||||
#include "dindexer-machinery/tiger.hpp"
|
||||
#include "pathname.hpp"
|
||||
#include <cassert>
|
||||
#include <boost/range/empty.hpp>
|
||||
|
|
@ -31,6 +32,18 @@
|
|||
|
||||
namespace mchlib {
|
||||
namespace {
|
||||
struct ProgressInfo {
|
||||
scantask::Hashing::ProgressCallback callback;
|
||||
boost::string_ref curr_path;
|
||||
uint64_t file_bytes_read;
|
||||
uint64_t total_bytes_read;
|
||||
uint32_t file_num;
|
||||
|
||||
void notify ( void ) {
|
||||
callback(curr_path, file_bytes_read, total_bytes_read, file_num);
|
||||
}
|
||||
};
|
||||
|
||||
void append_to_vec (std::vector<char>& parDest, const TigerHash& parHash, boost::string_ref parString) {
|
||||
const auto old_size = parDest.size();
|
||||
parDest.resize(old_size + sizeof(TigerHash) + parString.size());
|
||||
|
|
@ -44,7 +57,7 @@ namespace mchlib {
|
|||
std::copy(parString.begin(), parString.end(), parDest.begin() + old_size);
|
||||
}
|
||||
|
||||
void hash_dir (FileRecordData& parEntry, MutableSetListingView& parList, bool parIgnoreErrors) {
|
||||
void hash_dir (FileRecordData& parEntry, MutableSetListingView& parList, bool parIgnoreErrors, ProgressInfo& parProgressInfo) {
|
||||
assert(parEntry.is_directory);
|
||||
|
||||
//Build a blob with the hashes and filenames of every directory that
|
||||
|
|
@ -62,7 +75,7 @@ namespace mchlib {
|
|||
auto cd_list = MutableSetListingView(it);
|
||||
assert(boost::empty(cd_list) or cd_list.begin()->abs_path != it->abs_path);
|
||||
|
||||
hash_dir(*it, cd_list, parIgnoreErrors);
|
||||
hash_dir(*it, cd_list, parIgnoreErrors, parProgressInfo);
|
||||
append_to_vec(dir_blob, it->hash, basename);
|
||||
}
|
||||
else {
|
||||
|
|
@ -85,8 +98,13 @@ namespace mchlib {
|
|||
#endif
|
||||
//TODO: notify callback
|
||||
try {
|
||||
++parProgressInfo.file_num;
|
||||
parProgressInfo.curr_path = it->abs_path;
|
||||
parProgressInfo.notify();
|
||||
|
||||
tiger_file(it->abs_path, it->hash, parEntry.hash, it->size);
|
||||
it->hash_valid = true;
|
||||
parProgressInfo.total_bytes_read += it->size;
|
||||
}
|
||||
catch (const std::ios_base::failure& e) {
|
||||
if (parIgnoreErrors) {
|
||||
|
|
@ -104,11 +122,15 @@ namespace mchlib {
|
|||
#endif
|
||||
parEntry.hash_valid = true;
|
||||
}
|
||||
|
||||
void dummy_progress_callback (const boost::string_ref /*parPath*/, uint64_t /*parFileBytes*/, uint64_t /*parTotalBytes*/, uint32_t /*parFileNum*/) {
|
||||
}
|
||||
} //unnamed namespace
|
||||
|
||||
namespace scantask {
|
||||
Hashing::Hashing (std::shared_ptr<FileTreeBase> parFileTree, bool parIgnoreErrors) :
|
||||
m_file_tree_task(parFileTree),
|
||||
m_progress_callback(&dummy_progress_callback),
|
||||
m_ignore_errors(parIgnoreErrors)
|
||||
{
|
||||
assert(m_file_tree_task);
|
||||
|
|
@ -124,8 +146,24 @@ namespace mchlib {
|
|||
void Hashing::on_data_fill() {
|
||||
std::vector<FileRecordData>& file_list = m_file_tree_task->get_or_create();
|
||||
|
||||
ProgressInfo progr_info;
|
||||
progr_info.callback = m_progress_callback;
|
||||
progr_info.curr_path = "";
|
||||
progr_info.file_bytes_read = 0;
|
||||
progr_info.total_bytes_read = 0;
|
||||
progr_info.file_num = 0;
|
||||
|
||||
MutableSetListingView recordlist(file_list.begin(), file_list.end(), 0);
|
||||
hash_dir(file_list.front(), recordlist, m_ignore_errors);
|
||||
hash_dir(file_list.front(), recordlist, m_ignore_errors, progr_info);
|
||||
}
|
||||
|
||||
void Hashing::set_progress_callback (ProgressCallback parFunc) {
|
||||
if (parFunc) {
|
||||
m_progress_callback = parFunc;
|
||||
}
|
||||
else {
|
||||
m_progress_callback = &dummy_progress_callback;
|
||||
}
|
||||
}
|
||||
} //namespace scantask
|
||||
} //namespace mchlib
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ target_link_libraries(${PROJECT_NAME}
|
|||
PRIVATE ${bare_name}-if
|
||||
PRIVATE ${bare_name}-common
|
||||
PRIVATE ${bare_name}-machinery
|
||||
PRIVATE optimized pthread
|
||||
)
|
||||
|
||||
string(REPLACE "${bare_name}-" "" ACTION_NAME "${PROJECT_NAME}")
|
||||
|
|
|
|||
|
|
@ -45,10 +45,13 @@
|
|||
# include <condition_variable>
|
||||
#endif
|
||||
|
||||
//namespace {
|
||||
namespace {
|
||||
// void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
|
||||
// bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
|
||||
//} //unnamed namespace
|
||||
#if defined(WITH_PROGRESS_FEEDBACK)
|
||||
void print_progress ( const boost::string_ref parPath, uint64_t parFileBytes, uint64_t parTotalBytes, uint32_t parFileNum );
|
||||
#endif
|
||||
} //unnamed namespace
|
||||
|
||||
namespace stask = mchlib::scantask;
|
||||
|
||||
|
|
@ -94,9 +97,14 @@ int main (int parArgc, char* parArgv[]) {
|
|||
std::shared_ptr<FileRecordDataFiller> filerecdata(new FileRecordDataFiller(mime, hashing));
|
||||
std::shared_ptr<SetRecordDataFiller> setrecdata(new SetRecordDataFiller(media_type, content_type));
|
||||
|
||||
#if defined(WITH_PROGRESS_FEEDBACK)
|
||||
hashing->set_progress_callback(&print_progress);
|
||||
#endif
|
||||
|
||||
std::cout << "Content type: " << setrecdata->get_or_create().type << std::endl;
|
||||
|
||||
const auto& hashes = filerecdata->get_or_create();
|
||||
std::cout << std::endl;
|
||||
for (const auto& hash : hashes) {
|
||||
std::cout << '"' << hash.path <<
|
||||
"\" -> " << mchlib::tiger_to_string(hash.hash) <<
|
||||
|
|
@ -169,7 +177,7 @@ int main (int parArgc, char* parArgv[]) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
//namespace {
|
||||
namespace {
|
||||
// void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
|
||||
// if (parIndexer.empty()) {
|
||||
// return;
|
||||
|
|
@ -246,4 +254,11 @@ int main (int parArgc, char* parArgv[]) {
|
|||
// din::write_to_db(parDBSettings, parData, set_data, signature);
|
||||
// return true;
|
||||
// }
|
||||
//} //unnamed namespace
|
||||
|
||||
#if defined(WITH_PROGRESS_FEEDBACK)
|
||||
void print_progress (const boost::string_ref parPath, uint64_t /*parFileBytes*/, uint64_t parTotalBytes, uint32_t parFileNum) {
|
||||
std::cout << "Hashing file " << parFileNum << " \"" << parPath << "\" (" << parTotalBytes << " bytes hashed)\r";
|
||||
std::cout.flush();
|
||||
}
|
||||
#endif
|
||||
} //unnamed namespace
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue