diff --git a/cscope_gen.sh b/cscope_gen.sh index cb697d3..3f3265d 100755 --- a/cscope_gen.sh +++ b/cscope_gen.sh @@ -32,4 +32,4 @@ set -f find . \( $excl_paths -o $incl_extensions \) -a -type f $excl_files > cscope.files set +f -cscope -b -q +exec cscope -b -q diff --git a/include/dindexer-machinery/scantask/base.hpp b/include/dindexer-machinery/scantask/base.hpp new file mode 100644 index 0000000..ac889ab --- /dev/null +++ b/include/dindexer-machinery/scantask/base.hpp @@ -0,0 +1,68 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idCB253C1A5AFA46A18B8878ED4072CD96 +#define idCB253C1A5AFA46A18B8878ED4072CD96 + +#include + +namespace mchlib { + namespace scantask { + template + class Base { + protected: + Base ( void ); + virtual ~Base ( void ) noexcept = default; + + public: + T& get_or_create ( void ); + void clear_data ( void ); + + private: + virtual void on_data_destroy ( T& parData ) = 0; + virtual void on_data_create ( T& parData ) = 0; + + T m_data; + bool m_data_created; + }; + + template + Base::Base() : + m_data_created(false) + { + } + + template + T& Base::get_or_create() { + if (not m_data_created) { + m_data_created = true; + this->on_data_create(m_data); + } + return m_data; + } + + template + void Base::clear_data() { + if (m_data_created) { + m_data_created = false; + this->on_data_destroy(m_data); + } + } + } //namespace scantask +} //namespace mchlib + +#endif diff --git a/include/dindexer-machinery/scantask/dirtree.hpp b/include/dindexer-machinery/scantask/dirtree.hpp new file mode 100644 index 0000000..c1a1e73 --- /dev/null +++ b/include/dindexer-machinery/scantask/dirtree.hpp @@ -0,0 +1,45 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id0AA31B2E7D6244A08435CF9080E34AAE +#define id0AA31B2E7D6244A08435CF9080E34AAE + +#include "dindexer-machinery/scantask/base.hpp" +#include +#include + +namespace mchlib { + struct FileRecordData; + + namespace scantask { + class DirTree : public Base> { + public: + typedef std::vector PathList; + + explicit DirTree ( std::string parRoot ); + virtual ~DirTree ( void ) noexcept = default; + + private: + virtual void on_data_destroy ( PathList& parData ); + virtual void on_data_create ( PathList& parData ); + + std::string m_root; + }; + } //namespace scantask +} //namespace mchlib + +#endif diff --git a/src/machinery/CMakeLists.txt b/src/machinery/CMakeLists.txt index 35cc581..f678471 100644 --- a/src/machinery/CMakeLists.txt +++ b/src/machinery/CMakeLists.txt @@ -4,7 +4,6 @@ include(WithMediaAutodetect) find_package(Magic REQUIRED) add_library(${PROJECT_NAME} SHARED - indexer.cpp pathname.cpp tiger.c tiger.cpp @@ -16,6 +15,7 @@ add_library(${PROJECT_NAME} SHARED guess_content_type.cpp set_listing.cpp globbing.cpp + scantask/dirtree.cpp ) #target_include_directories(${PROJECT_NAME} @@ -29,6 +29,10 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ${MAGIC_LIBRARIES} ) +target_include_directories(${PROJECT_NAME} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} +) + if (DINDEXER_WITH_MEDIA_AUTODETECT) target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${BLKID_INCLUDE_DIRS} diff --git a/src/machinery/filesearcher.cpp b/src/machinery/filesearcher.cpp index 2ac6133..499c9c1 100644 --- a/src/machinery/filesearcher.cpp +++ b/src/machinery/filesearcher.cpp @@ -15,7 +15,7 @@ * along with "dindexer". If not, see . */ -#include "dindexer-machinery/filesearcher.hpp" +#include "filesearcher.hpp" #if !defined(_XOPEN_SOURCE) #define _XOPEN_SOURCE 500 diff --git a/include/dindexer-machinery/filesearcher.hpp b/src/machinery/filesearcher.hpp similarity index 100% rename from include/dindexer-machinery/filesearcher.hpp rename to src/machinery/filesearcher.hpp diff --git a/include/dindexer-machinery/filestats.hpp b/src/machinery/filestats.hpp similarity index 100% rename from include/dindexer-machinery/filestats.hpp rename to src/machinery/filestats.hpp diff --git a/src/machinery/scantask/dirtree.cpp b/src/machinery/scantask/dirtree.cpp new file mode 100644 index 0000000..992002a --- /dev/null +++ b/src/machinery/scantask/dirtree.cpp @@ -0,0 +1,81 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "dindexer-machinery/scantask/dirtree.hpp" +#include "dindexer-machinery/recorddata.hpp" +#include "dindexer-machinery/set_listing.hpp" +#include "filesearcher.hpp" +#include +#include +#include +#include + +namespace mchlib { + namespace { + FileRecordData make_file_record_data (const char* parPath, const fastf::FileStats& parSt) { + return FileRecordData( + parPath, + parSt.atime, + parSt.mtime, + parSt.level, + parSt.is_dir, + parSt.is_symlink + ); + } + + bool add_path (scantask::DirTree::PathList& parOut, const char* parPath, const fastf::FileStats& parStats) { + auto it_before = SetListing::lower_bound( + parOut, + parPath, + parStats.level, + parStats.is_dir + ); + + parOut.insert( + it_before, + make_file_record_data(parPath, parStats) + ); + return true; + } + } + + namespace scantask { + DirTree::DirTree (std::string parRoot) : + m_root(std::move(parRoot)) + { + assert(not m_root.empty()); + } + + void DirTree::on_data_destroy (PathList& parData) { + parData.clear(); + } + + void DirTree::on_data_create (PathList& parData) { + using std::placeholders::_1; + using std::placeholders::_2; + + assert(parData.empty()); + + fastf::FileSearcher searcher(m_root); + fastf::FileSearcher::ConstCharVecType ext, ignore; + + searcher.SetFollowSymlinks(true); + searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&add_path, std::ref(parData), _1, _2))); + searcher.Search(ext, ignore); + } + } //namespace scantask +} //namespace mchlib diff --git a/src/scan/main.cpp b/src/scan/main.cpp index 561598c..533e484 100644 --- a/src/scan/main.cpp +++ b/src/scan/main.cpp @@ -21,14 +21,13 @@ #include "dindexer-machinery/recorddata.hpp" #include "dindexerConfig.h" -#include "dindexer-machinery/filesearcher.hpp" -#include "dindexer-machinery/indexer.hpp" #include "dindexer-machinery/machinery_info.hpp" #include "dindexer-common/common_info.hpp" #include "dindexer-common/settings.hpp" -#include "dindexer-machinery/guess_content_type.hpp" +//#include "dindexer-machinery/guess_content_type.hpp" #include "commandline.hpp" #include "dbbackend.hpp" +#include "dindexer-machinery/scantask/dirtree.hpp" #include #include #include @@ -41,10 +40,10 @@ # include #endif -namespace { - void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress ); - bool add_to_db ( const std::vector& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false ); -} //unnamed namespace +//namespace { +// void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress ); +// bool add_to_db ( const std::vector& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false ); +//} //unnamed namespace int main (int parArgc, char* parArgv[]) { using std::placeholders::_1; @@ -61,7 +60,6 @@ int main (int parArgc, char* parArgv[]) { std::cerr << err.what() << "\nUse --help for help" << std::endl; return 2; } - const std::string search_path(vm["search-path"].as()); #if defined(WITH_PROGRESS_FEEDBACK) const bool verbose = (0 == vm.count("quiet")); #else @@ -77,48 +75,49 @@ int main (int parArgc, char* parArgv[]) { } } + const std::string search_path(vm["search-path"].as()); + mchlib::scantask::DirTree scan_dirtree(search_path); + #if defined(WITH_MEDIA_AUTODETECT) - char set_type; - if (0 == vm.count("type")) { - std::cout << "Analyzing disc... "; - try { - const auto guessed_type = mchlib::guess_media_type(std::string(search_path)); - set_type = guessed_type; - std::cout << "Setting type to " << set_type << " (" - << dinlib::media_type_to_str(guessed_type) << ")\n"; - } - catch (const std::runtime_error& e) { - std::cout << '\n'; - std::cerr << e.what(); - return 1; - } - } - else { - set_type = vm["type"].as(); - } + //char set_type; + //if (0 == vm.count("type")) { + // std::cout << "Analyzing disc... "; + // try { + // const auto guessed_type = mchlib::guess_media_type(std::string(search_path)); + // set_type = guessed_type; + // std::cout << "Setting type to " << set_type << " (" + // << dinlib::media_type_to_str(guessed_type) << ")\n"; + // } + // catch (const std::runtime_error& e) { + // std::cout << '\n'; + // std::cerr << e.what(); + // return 1; + // } + //} + //else { + // set_type = vm["type"].as(); + //} + //std::unique_ptr media_autodetector( + //new mchlib::scantask::MediaAutodetect(vm["type"].as()) + //); #else const char set_type = vm["type"].as(); #endif std::cout << "constructing...\n"; - mchlib::Indexer indexer; - indexer.ignore_read_errors(vm.count("ignore-errors") > 0); - fastf::FileSearcher searcher(search_path); - fastf::FileSearcher::ConstCharVecType ext, ignore; - searcher.SetFollowSymlinks(true); - searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&mchlib::Indexer::add_path, &indexer, _1, _2))); - searcher.Search(ext, ignore); - if (verbose) { - std::cout << "Fetching items list...\n"; - } + //indexer.ignore_read_errors(vm.count("ignore-errors") > 0); + //if (verbose) { + // std::cout << "Fetching items list...\n"; + //} - if (indexer.empty()) { - std::cerr << "Nothing found at the given location, quitting\n"; - return 1; - } - else { - run_hash_calculation(indexer, verbose); + //if (indexer.empty()) { + // std::cerr << "Nothing found at the given location, quitting\n"; + // return 1; + //} + //else { + { + //run_hash_calculation(indexer, verbose); //TODO: guess_content_type() relies on FileRecordData::path being set to //the relative path already. Unfortunately at this point it just got @@ -128,95 +127,95 @@ int main (int parArgc, char* parArgv[]) { //paths are populated at the end of calculate_hash(), so until I come up //with a better system I'm just moving content detection to after hash //calculation. - const auto set_type_casted = dinlib::char_to_media_type(set_type); - const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data()); - const char content_type = mchlib::content_type_to_char(content); + //const auto set_type_casted = dinlib::char_to_media_type(set_type); + //const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data()); + //const char content_type = mchlib::content_type_to_char(content); if (verbose) { std::cout << "Writing to database...\n"; } - if (not add_to_db(indexer.record_data(), vm["setname"].as(), set_type, content_type, settings.db)) { - std::cerr << "Not written to DB, likely because a set with the same hash already exists\n"; - } + //if (not add_to_db(indexer.record_data(), vm["setname"].as(), set_type, content_type, settings.db)) { + // std::cerr << "Not written to DB, likely because a set with the same hash already exists\n"; + //} } return 0; } -namespace { - void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) { - if (parIndexer.empty()) { - return; - } - -#if !defined(WITH_PROGRESS_FEEDBACK) - parShowProgress = false; -#endif - if (not parShowProgress) { -//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1 -//terminate called after throwing an instance of 'std::ios_base::failure' -//what(): basic_filebuf::xsgetn error reading the file -//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato - parIndexer.calculate_hash(); - } -#if defined(WITH_PROGRESS_FEEDBACK) - else { - typedef std::ostream_iterator cout_iterator; - - std::cout << "Processing"; - std::cout.flush(); - const auto total_items = parIndexer.total_items(); - std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer); - std::mutex progress_print; - std::size_t clear_size = 0; - const auto digit_count = static_cast(std::log10(static_cast(total_items))) + 1; - do { - //TODO: fix this steaming pile of crap - //std::unique_lock lk(progress_print); - //parIndexer.step_notify().wait(lk); - std::cout << '\r'; - std::fill_n(cout_iterator(std::cout), clear_size, ' '); - std::cout << '\r'; - { - std::ostringstream oss; - const auto item_index = std::min(total_items - 1, parIndexer.processed_items()); - oss << "Processing file " - << std::setw(digit_count) << std::setfill(' ') << (item_index + 1) - << " of " << total_items << " \"" << parIndexer.current_item() << '"'; - const auto msg(oss.str()); - clear_size = msg.size(); - std::cout << msg; - std::cout.flush(); - } - } while (false); //parIndexer.processed_items() != total_items); - - hash_thread.join(); - if (parIndexer.processed_items() > 0) { - std::cout << '\n'; - } - } -#endif - } - - bool add_to_db (const std::vector& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) { - using mchlib::FileRecordData; - using mchlib::SetRecordDataFull; - using mchlib::SetRecordData; - - if (not parForce) { - const auto& first_hash = parData.front().hash; - FileRecordData itm; - SetRecordDataFull set; - const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash); - if (already_in_db) { - return false; - } - } - - SetRecordData set_data {parSetName, parType, parContentType }; - const auto app_signature = dinlib::dindexer_signature(); - const auto lib_signature = mchlib::lib_signature(); - const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size()); - din::write_to_db(parDBSettings, parData, set_data, signature); - return true; - } -} //unnamed namespace +//namespace { +// void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) { +// if (parIndexer.empty()) { +// return; +// } +// +//#if !defined(WITH_PROGRESS_FEEDBACK) +// parShowProgress = false; +//#endif +// if (not parShowProgress) { +////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1 +////terminate called after throwing an instance of 'std::ios_base::failure' +////what(): basic_filebuf::xsgetn error reading the file +////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato +// parIndexer.calculate_hash(); +// } +//#if defined(WITH_PROGRESS_FEEDBACK) +// else { +// typedef std::ostream_iterator cout_iterator; +// +// std::cout << "Processing"; +// std::cout.flush(); +// const auto total_items = parIndexer.total_items(); +// std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer); +// std::mutex progress_print; +// std::size_t clear_size = 0; +// const auto digit_count = static_cast(std::log10(static_cast(total_items))) + 1; +// do { +// //TODO: fix this steaming pile of crap +// //std::unique_lock lk(progress_print); +// //parIndexer.step_notify().wait(lk); +// std::cout << '\r'; +// std::fill_n(cout_iterator(std::cout), clear_size, ' '); +// std::cout << '\r'; +// { +// std::ostringstream oss; +// const auto item_index = std::min(total_items - 1, parIndexer.processed_items()); +// oss << "Processing file " +// << std::setw(digit_count) << std::setfill(' ') << (item_index + 1) +// << " of " << total_items << " \"" << parIndexer.current_item() << '"'; +// const auto msg(oss.str()); +// clear_size = msg.size(); +// std::cout << msg; +// std::cout.flush(); +// } +// } while (false); //parIndexer.processed_items() != total_items); +// +// hash_thread.join(); +// if (parIndexer.processed_items() > 0) { +// std::cout << '\n'; +// } +// } +//#endif +// } +// +// bool add_to_db (const std::vector& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) { +// using mchlib::FileRecordData; +// using mchlib::SetRecordDataFull; +// using mchlib::SetRecordData; +// +// if (not parForce) { +// const auto& first_hash = parData.front().hash; +// FileRecordData itm; +// SetRecordDataFull set; +// const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash); +// if (already_in_db) { +// return false; +// } +// } +// +// SetRecordData set_data {parSetName, parType, parContentType }; +// const auto app_signature = dinlib::dindexer_signature(); +// const auto lib_signature = mchlib::lib_signature(); +// const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size()); +// din::write_to_db(parDBSettings, parData, set_data, signature); +// return true; +// } +//} //unnamed namespace