King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-02-19 12:04:54 +00:00
Code Issues Releases Wiki Activity

Create DirTree class and start moving code around.

Browse source
This commit is contained in:
King_DuckZ 2016-03-03 20:46:30 +01:00
parent c69d17395a
commit 5b3e15c45f
9 changed files with 326 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cscope_gen.sh
View file

@ -32,4 +32,4 @@ set -f
find . \( $excl_paths -o $incl_extensions \) -a -type f $excl_files > cscope.files
set +f
cscope -b -q
exec cscope -b -q

68
include/dindexer-machinery/scantask/base.hpp Normal file
View file

@ -0,0 +1,68 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef idCB253C1A5AFA46A18B8878ED4072CD96
#define idCB253C1A5AFA46A18B8878ED4072CD96
#include <ciso646>
namespace mchlib {
namespace scantask {
template <typename T>
class Base {
protected:
Base ( void );
virtual ~Base ( void ) noexcept = default;
public:
T& get_or_create ( void );
void clear_data ( void );
private:
virtual void on_data_destroy ( T& parData ) = 0;
virtual void on_data_create ( T& parData ) = 0;
T m_data;
bool m_data_created;
};
template <typename T>
Base<T>::Base() :
m_data_created(false)
{
}
template <typename T>
T& Base<T>::get_or_create() {
if (not m_data_created) {
m_data_created = true;
this->on_data_create(m_data);
}
return m_data;
}
template <typename T>
void Base<T>::clear_data() {
if (m_data_created) {
m_data_created = false;
this->on_data_destroy(m_data);
}
}
} //namespace scantask
} //namespace mchlib
#endif

45
include/dindexer-machinery/scantask/dirtree.hpp Normal file
View file

@ -0,0 +1,45 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id0AA31B2E7D6244A08435CF9080E34AAE
#define id0AA31B2E7D6244A08435CF9080E34AAE
#include "dindexer-machinery/scantask/base.hpp"
#include <string>
#include <vector>
namespace mchlib {
struct FileRecordData;
namespace scantask {
class DirTree : public Base<std::vector<FileRecordData>> {
public:
typedef std::vector<FileRecordData> PathList;
explicit DirTree ( std::string parRoot );
virtual ~DirTree ( void ) noexcept = default;
private:
virtual void on_data_destroy ( PathList& parData );
virtual void on_data_create ( PathList& parData );
std::string m_root;
};
} //namespace scantask
} //namespace mchlib
#endif

6
src/machinery/CMakeLists.txt
View file

@ -4,7 +4,6 @@ include(WithMediaAutodetect)
find_package(Magic REQUIRED)
add_library(${PROJECT_NAME} SHARED
indexer.cpp
pathname.cpp
tiger.c
tiger.cpp
@ -16,6 +15,7 @@ add_library(${PROJECT_NAME} SHARED
guess_content_type.cpp
set_listing.cpp
globbing.cpp
scantask/dirtree.cpp
)
#target_include_directories(${PROJECT_NAME}
@ -29,6 +29,10 @@ target_link_libraries(${PROJECT_NAME}
PRIVATE ${MAGIC_LIBRARIES}
)
target_include_directories(${PROJECT_NAME}
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
)
if (DINDEXER_WITH_MEDIA_AUTODETECT)
target_include_directories(${PROJECT_NAME} SYSTEM
PRIVATE ${BLKID_INCLUDE_DIRS}

2
src/machinery/filesearcher.cpp
View file

@ -15,7 +15,7 @@
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "dindexer-machinery/filesearcher.hpp"
#include "filesearcher.hpp"
#if !defined(_XOPEN_SOURCE)
#define _XOPEN_SOURCE 500

0
include/dindexer-machinery/filesearcher.hpp → src/machinery/filesearcher.hpp
View file

0
include/dindexer-machinery/filestats.hpp → src/machinery/filestats.hpp
View file

81
src/machinery/scantask/dirtree.cpp Normal file
View file

@ -0,0 +1,81 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "dindexer-machinery/scantask/dirtree.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include "dindexer-machinery/set_listing.hpp"
#include "filesearcher.hpp"
#include <utility>
#include <cassert>
#include <ciso646>
#include <functional>
namespace mchlib {
namespace {
FileRecordData make_file_record_data (const char* parPath, const fastf::FileStats& parSt) {
return FileRecordData(
parPath,
parSt.atime,
parSt.mtime,
parSt.level,
parSt.is_dir,
parSt.is_symlink
);
}
bool add_path (scantask::DirTree::PathList& parOut, const char* parPath, const fastf::FileStats& parStats) {
auto it_before = SetListing::lower_bound(
parOut,
parPath,
parStats.level,
parStats.is_dir
);
parOut.insert(
it_before,
make_file_record_data(parPath, parStats)
);
return true;
}
}
namespace scantask {
DirTree::DirTree (std::string parRoot) :
m_root(std::move(parRoot))
{
assert(not m_root.empty());
}
void DirTree::on_data_destroy (PathList& parData) {
parData.clear();
}
void DirTree::on_data_create (PathList& parData) {
using std::placeholders::_1;
using std::placeholders::_2;
assert(parData.empty());
fastf::FileSearcher searcher(m_root);
fastf::FileSearcher::ConstCharVecType ext, ignore;
searcher.SetFollowSymlinks(true);
searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&add_path, std::ref(parData), _1, _2)));
searcher.Search(ext, ignore);
}
} //namespace scantask
} //namespace mchlib

251
src/scan/main.cpp
View file

@ -21,14 +21,13 @@
#include "dindexer-machinery/recorddata.hpp"
#include "dindexerConfig.h"
#include "dindexer-machinery/filesearcher.hpp"
#include "dindexer-machinery/indexer.hpp"
#include "dindexer-machinery/machinery_info.hpp"
#include "dindexer-common/common_info.hpp"
#include "dindexer-common/settings.hpp"
#include "dindexer-machinery/guess_content_type.hpp"
//#include "dindexer-machinery/guess_content_type.hpp"
#include "commandline.hpp"
#include "dbbackend.hpp"
#include "dindexer-machinery/scantask/dirtree.hpp"
#include <iostream>
#include <iomanip>
#include <ciso646>
@ -41,10 +40,10 @@
# include <condition_variable>
#endif
namespace {
void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
} //unnamed namespace
//namespace {
// void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
// bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
//} //unnamed namespace
int main (int parArgc, char* parArgv[]) {
using std::placeholders::_1;
@ -61,7 +60,6 @@ int main (int parArgc, char* parArgv[]) {
std::cerr << err.what() << "\nUse --help for help" << std::endl;
return 2;
}
const std::string search_path(vm["search-path"].as<std::string>());
#if defined(WITH_PROGRESS_FEEDBACK)
const bool verbose = (0 == vm.count("quiet"));
#else
@ -77,48 +75,49 @@ int main (int parArgc, char* parArgv[]) {
}
}
const std::string search_path(vm["search-path"].as<std::string>());
mchlib::scantask::DirTree scan_dirtree(search_path);
#if defined(WITH_MEDIA_AUTODETECT)
char set_type;
if (0 == vm.count("type")) {
std::cout << "Analyzing disc... ";
try {
const auto guessed_type = mchlib::guess_media_type(std::string(search_path));
set_type = guessed_type;
std::cout << "Setting type to " << set_type << " ("
<< dinlib::media_type_to_str(guessed_type) << ")\n";
}
catch (const std::runtime_error& e) {
std::cout << '\n';
std::cerr << e.what();
return 1;
}
}
else {
set_type = vm["type"].as<char>();
}
//char set_type;
//if (0 == vm.count("type")) {
// std::cout << "Analyzing disc... ";
// try {
// const auto guessed_type = mchlib::guess_media_type(std::string(search_path));
// set_type = guessed_type;
// std::cout << "Setting type to " << set_type << " ("
// << dinlib::media_type_to_str(guessed_type) << ")\n";
// }
// catch (const std::runtime_error& e) {
// std::cout << '\n';
// std::cerr << e.what();
// return 1;
// }
//}
//else {
// set_type = vm["type"].as<char>();
//}
//std::unique_ptr<mchlib::scantask::MediaAutodetect> media_autodetector(
//new mchlib::scantask::MediaAutodetect(vm["type"].as<char>())
//);
#else
const char set_type = vm["type"].as<char>();
#endif
std::cout << "constructing...\n";
mchlib::Indexer indexer;
indexer.ignore_read_errors(vm.count("ignore-errors") > 0);
fastf::FileSearcher searcher(search_path);
fastf::FileSearcher::ConstCharVecType ext, ignore;
searcher.SetFollowSymlinks(true);
searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&mchlib::Indexer::add_path, &indexer, _1, _2)));
searcher.Search(ext, ignore);
if (verbose) {
std::cout << "Fetching items list...\n";
}
//indexer.ignore_read_errors(vm.count("ignore-errors") > 0);
//if (verbose) {
// std::cout << "Fetching items list...\n";
//}
if (indexer.empty()) {
std::cerr << "Nothing found at the given location, quitting\n";
return 1;
}
else {
run_hash_calculation(indexer, verbose);
//if (indexer.empty()) {
// std::cerr << "Nothing found at the given location, quitting\n";
// return 1;
//}
//else {
{
//run_hash_calculation(indexer, verbose);
//TODO: guess_content_type() relies on FileRecordData::path being set to
//the relative path already. Unfortunately at this point it just got
@ -128,95 +127,95 @@ int main (int parArgc, char* parArgv[]) {
//paths are populated at the end of calculate_hash(), so until I come up
//with a better system I'm just moving content detection to after hash
//calculation.
const auto set_type_casted = dinlib::char_to_media_type(set_type);
const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data());
const char content_type = mchlib::content_type_to_char(content);
//const auto set_type_casted = dinlib::char_to_media_type(set_type);
//const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data());
//const char content_type = mchlib::content_type_to_char(content);
if (verbose) {
std::cout << "Writing to database...\n";
}
if (not add_to_db(indexer.record_data(), vm["setname"].as<std::string>(), set_type, content_type, settings.db)) {
std::cerr << "Not written to DB, likely because a set with the same hash already exists\n";
}
//if (not add_to_db(indexer.record_data(), vm["setname"].as<std::string>(), set_type, content_type, settings.db)) {
// std::cerr << "Not written to DB, likely because a set with the same hash already exists\n";
//}
}
return 0;
}
namespace {
void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
if (parIndexer.empty()) {
return;
}
#if !defined(WITH_PROGRESS_FEEDBACK)
parShowProgress = false;
#endif
if (not parShowProgress) {
//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1
//terminate called after throwing an instance of 'std::ios_base::failure'
//what(): basic_filebuf::xsgetn error reading the file
//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato
parIndexer.calculate_hash();
}
#if defined(WITH_PROGRESS_FEEDBACK)
else {
typedef std::ostream_iterator<char> cout_iterator;
std::cout << "Processing";
std::cout.flush();
const auto total_items = parIndexer.total_items();
std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer);
std::mutex progress_print;
std::size_t clear_size = 0;
const auto digit_count = static_cast<std::size_t>(std::log10(static_cast<double>(total_items))) + 1;
do {
//TODO: fix this steaming pile of crap
//std::unique_lock<std::mutex> lk(progress_print);
//parIndexer.step_notify().wait(lk);
std::cout << '\r';
std::fill_n(cout_iterator(std::cout), clear_size, ' ');
std::cout << '\r';
{
std::ostringstream oss;
const auto item_index = std::min(total_items - 1, parIndexer.processed_items());
oss << "Processing file "
<< std::setw(digit_count) << std::setfill(' ') << (item_index + 1)
<< " of " << total_items << " \"" << parIndexer.current_item() << '"';
const auto msg(oss.str());
clear_size = msg.size();
std::cout << msg;
std::cout.flush();
}
} while (false); //parIndexer.processed_items() != total_items);
hash_thread.join();
if (parIndexer.processed_items() > 0) {
std::cout << '\n';
}
}
#endif
}
bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) {
using mchlib::FileRecordData;
using mchlib::SetRecordDataFull;
using mchlib::SetRecordData;
if (not parForce) {
const auto& first_hash = parData.front().hash;
FileRecordData itm;
SetRecordDataFull set;
const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash);
if (already_in_db) {
return false;
}
}
SetRecordData set_data {parSetName, parType, parContentType };
const auto app_signature = dinlib::dindexer_signature();
const auto lib_signature = mchlib::lib_signature();
const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
din::write_to_db(parDBSettings, parData, set_data, signature);
return true;
}
} //unnamed namespace
//namespace {
// void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
// if (parIndexer.empty()) {
// return;
// }
//
//#if !defined(WITH_PROGRESS_FEEDBACK)
// parShowProgress = false;
//#endif
// if (not parShowProgress) {
////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1
////terminate called after throwing an instance of 'std::ios_base::failure'
////what(): basic_filebuf::xsgetn error reading the file
////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato
// parIndexer.calculate_hash();
// }
//#if defined(WITH_PROGRESS_FEEDBACK)
// else {
// typedef std::ostream_iterator<char> cout_iterator;
//
// std::cout << "Processing";
// std::cout.flush();
// const auto total_items = parIndexer.total_items();
// std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer);
// std::mutex progress_print;
// std::size_t clear_size = 0;
// const auto digit_count = static_cast<std::size_t>(std::log10(static_cast<double>(total_items))) + 1;
// do {
// //TODO: fix this steaming pile of crap
// //std::unique_lock<std::mutex> lk(progress_print);
// //parIndexer.step_notify().wait(lk);
// std::cout << '\r';
// std::fill_n(cout_iterator(std::cout), clear_size, ' ');
// std::cout << '\r';
// {
// std::ostringstream oss;
// const auto item_index = std::min(total_items - 1, parIndexer.processed_items());
// oss << "Processing file "
// << std::setw(digit_count) << std::setfill(' ') << (item_index + 1)
// << " of " << total_items << " \"" << parIndexer.current_item() << '"';
// const auto msg(oss.str());
// clear_size = msg.size();
// std::cout << msg;
// std::cout.flush();
// }
// } while (false); //parIndexer.processed_items() != total_items);
//
// hash_thread.join();
// if (parIndexer.processed_items() > 0) {
// std::cout << '\n';
// }
// }
//#endif
// }
//
// bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) {
// using mchlib::FileRecordData;
// using mchlib::SetRecordDataFull;
// using mchlib::SetRecordData;
//
// if (not parForce) {
// const auto& first_hash = parData.front().hash;
// FileRecordData itm;
// SetRecordDataFull set;
// const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash);
// if (already_in_db) {
// return false;
// }
// }
//
// SetRecordData set_data {parSetName, parType, parContentType };
// const auto app_signature = dinlib::dindexer_signature();
// const auto lib_signature = mchlib::lib_signature();
// const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
// din::write_to_db(parDBSettings, parData, set_data, signature);
// return true;
// }
//} //unnamed namespace