1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-02-17 11:45:50 +00:00

Re-add some progress feedback.

It's not multithreaded this time o.O
This commit is contained in:
King_DuckZ 2016-03-10 09:47:55 +01:00
parent c28398a1b3
commit 43c8024b0c
4 changed files with 67 additions and 9 deletions

View file

@ -19,9 +19,11 @@
#define idC7CC55298AC049EAA80604D6C7FD081D
#include "dindexer-machinery/scantask/leanbase.hpp"
#include "dindexer-machinery/tiger.hpp"
#include <vector>
#include <memory>
#include <functional>
#include <boost/utility/string_ref.hpp>
#include <cstdint>
namespace mchlib {
struct FileRecordData;
@ -30,15 +32,19 @@ namespace mchlib {
class Hashing : public LeanBase<std::vector<FileRecordData>> {
public:
typedef LeanBase<std::vector<FileRecordData>> FileTreeBase;
typedef std::function<void(const boost::string_ref, uint64_t, uint64_t, uint32_t)> ProgressCallback;
Hashing ( std::shared_ptr<FileTreeBase> parFileTree, bool parIgnoreErrors );
virtual ~Hashing ( void ) noexcept;
void set_progress_callback ( ProgressCallback parFunc );
private:
virtual void on_data_fill ( void ) override;
virtual std::vector<FileRecordData>& on_data_get ( void ) override;
std::shared_ptr<FileTreeBase> m_file_tree_task;
ProgressCallback m_progress_callback;
bool m_ignore_errors;
};
} //namespace scantask

View file

@ -18,6 +18,7 @@
#include "dindexer-machinery/scantask/hashing.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include "dindexer-machinery/set_listing.hpp"
#include "dindexer-machinery/tiger.hpp"
#include "pathname.hpp"
#include <cassert>
#include <boost/range/empty.hpp>
@ -31,6 +32,18 @@
namespace mchlib {
namespace {
struct ProgressInfo {
scantask::Hashing::ProgressCallback callback;
boost::string_ref curr_path;
uint64_t file_bytes_read;
uint64_t total_bytes_read;
uint32_t file_num;
void notify ( void ) {
callback(curr_path, file_bytes_read, total_bytes_read, file_num);
}
};
void append_to_vec (std::vector<char>& parDest, const TigerHash& parHash, boost::string_ref parString) {
const auto old_size = parDest.size();
parDest.resize(old_size + sizeof(TigerHash) + parString.size());
@ -44,7 +57,7 @@ namespace mchlib {
std::copy(parString.begin(), parString.end(), parDest.begin() + old_size);
}
void hash_dir (FileRecordData& parEntry, MutableSetListingView& parList, bool parIgnoreErrors) {
void hash_dir (FileRecordData& parEntry, MutableSetListingView& parList, bool parIgnoreErrors, ProgressInfo& parProgressInfo) {
assert(parEntry.is_directory);
//Build a blob with the hashes and filenames of every directory that
@ -62,7 +75,7 @@ namespace mchlib {
auto cd_list = MutableSetListingView(it);
assert(boost::empty(cd_list) or cd_list.begin()->abs_path != it->abs_path);
hash_dir(*it, cd_list, parIgnoreErrors);
hash_dir(*it, cd_list, parIgnoreErrors, parProgressInfo);
append_to_vec(dir_blob, it->hash, basename);
}
else {
@ -85,8 +98,13 @@ namespace mchlib {
#endif
//TODO: notify callback
try {
++parProgressInfo.file_num;
parProgressInfo.curr_path = it->abs_path;
parProgressInfo.notify();
tiger_file(it->abs_path, it->hash, parEntry.hash, it->size);
it->hash_valid = true;
parProgressInfo.total_bytes_read += it->size;
}
catch (const std::ios_base::failure& e) {
if (parIgnoreErrors) {
@ -104,11 +122,15 @@ namespace mchlib {
#endif
parEntry.hash_valid = true;
}
void dummy_progress_callback (const boost::string_ref /*parPath*/, uint64_t /*parFileBytes*/, uint64_t /*parTotalBytes*/, uint32_t /*parFileNum*/) {
}
} //unnamed namespace
namespace scantask {
Hashing::Hashing (std::shared_ptr<FileTreeBase> parFileTree, bool parIgnoreErrors) :
m_file_tree_task(parFileTree),
m_progress_callback(&dummy_progress_callback),
m_ignore_errors(parIgnoreErrors)
{
assert(m_file_tree_task);
@ -124,8 +146,24 @@ namespace mchlib {
void Hashing::on_data_fill() {
std::vector<FileRecordData>& file_list = m_file_tree_task->get_or_create();
ProgressInfo progr_info;
progr_info.callback = m_progress_callback;
progr_info.curr_path = "";
progr_info.file_bytes_read = 0;
progr_info.total_bytes_read = 0;
progr_info.file_num = 0;
MutableSetListingView recordlist(file_list.begin(), file_list.end(), 0);
hash_dir(file_list.front(), recordlist, m_ignore_errors);
hash_dir(file_list.front(), recordlist, m_ignore_errors, progr_info);
}
void Hashing::set_progress_callback (ProgressCallback parFunc) {
if (parFunc) {
m_progress_callback = parFunc;
}
else {
m_progress_callback = &dummy_progress_callback;
}
}
} //namespace scantask
} //namespace mchlib

View file

@ -16,7 +16,6 @@ target_link_libraries(${PROJECT_NAME}
PRIVATE ${bare_name}-if
PRIVATE ${bare_name}-common
PRIVATE ${bare_name}-machinery
PRIVATE optimized pthread
)
string(REPLACE "${bare_name}-" "" ACTION_NAME "${PROJECT_NAME}")

View file

@ -45,10 +45,13 @@
# include <condition_variable>
#endif
//namespace {
namespace {
// void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
// bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
//} //unnamed namespace
#if defined(WITH_PROGRESS_FEEDBACK)
void print_progress ( const boost::string_ref parPath, uint64_t parFileBytes, uint64_t parTotalBytes, uint32_t parFileNum );
#endif
} //unnamed namespace
namespace stask = mchlib::scantask;
@ -94,9 +97,14 @@ int main (int parArgc, char* parArgv[]) {
std::shared_ptr<FileRecordDataFiller> filerecdata(new FileRecordDataFiller(mime, hashing));
std::shared_ptr<SetRecordDataFiller> setrecdata(new SetRecordDataFiller(media_type, content_type));
#if defined(WITH_PROGRESS_FEEDBACK)
hashing->set_progress_callback(&print_progress);
#endif
std::cout << "Content type: " << setrecdata->get_or_create().type << std::endl;
const auto& hashes = filerecdata->get_or_create();
std::cout << std::endl;
for (const auto& hash : hashes) {
std::cout << '"' << hash.path <<
"\" -> " << mchlib::tiger_to_string(hash.hash) <<
@ -169,7 +177,7 @@ int main (int parArgc, char* parArgv[]) {
return 0;
}
//namespace {
namespace {
// void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
// if (parIndexer.empty()) {
// return;
@ -246,4 +254,11 @@ int main (int parArgc, char* parArgv[]) {
// din::write_to_db(parDBSettings, parData, set_data, signature);
// return true;
// }
//} //unnamed namespace
#if defined(WITH_PROGRESS_FEEDBACK)
void print_progress (const boost::string_ref parPath, uint64_t /*parFileBytes*/, uint64_t parTotalBytes, uint32_t parFileNum) {
std::cout << "Hashing file " << parFileNum << " \"" << parPath << "\" (" << parTotalBytes << " bytes hashed)\r";
std::cout.flush();
}
#endif
} //unnamed namespace