1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-10-20 17:19:23 +00:00

Allow passing set name and search path as command line parameters.

This commit is contained in:
King_DuckZ 2015-11-11 19:24:10 +00:00
commit 585c7f45b7
13 changed files with 228 additions and 36 deletions

View file

@ -1,13 +1,19 @@
cmake_minimum_required(VERSION 3.0 FATAL_ERROR) cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(dindexer VERSION 0.1 LANGUAGES CXX C) project(dindexer VERSION 0.1.0 LANGUAGES CXX C)
list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules) list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules)
set(PROJECT_VERSION_BETA "1")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++11") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++11")
find_library(Boost REQUIRED 1.53.0) find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options)
find_package(PostgreSQL REQUIRED) find_package(PostgreSQL REQUIRED)
find_package(YamlCpp REQUIRED 0.5.1) find_package(YamlCpp 0.5.1 REQUIRED)
configure_file(
"${PROJECT_SOURCE_DIR}/src/${PROJECT_NAME}Config.h.in"
"${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.h"
)
add_executable(${PROJECT_NAME} add_executable(${PROJECT_NAME}
src/main.cpp src/main.cpp
@ -21,10 +27,11 @@ add_executable(${PROJECT_NAME}
src/pq/resultset.cpp src/pq/resultset.cpp
src/dbbackend.cpp src/dbbackend.cpp
src/settings.cpp src/settings.cpp
src/commandline.cpp
) )
target_include_directories(${PROJECT_NAME} SYSTEM target_include_directories(${PROJECT_NAME} SYSTEM
INTERFACE ${Boost_INCLUDE_DIR} INTERFACE ${Boost_INCLUDE_DIRS}
PRIVATE ${PostgreSQL_INCLUDE_DIRS} PRIVATE ${PostgreSQL_INCLUDE_DIRS}
PRIVATE ${YAMLCPP_INCLUDE_DIR} PRIVATE ${YAMLCPP_INCLUDE_DIR}
) )
@ -32,9 +39,11 @@ target_include_directories(${PROJECT_NAME} SYSTEM
target_include_directories(${PROJECT_NAME} target_include_directories(${PROJECT_NAME}
PRIVATE src PRIVATE src
PRIVATE include PRIVATE include
PRIVATE ${PROJECT_BINARY_DIR}
) )
target_link_libraries(${PROJECT_NAME} target_link_libraries(${PROJECT_NAME}
${PostgreSQL_LIBRARIES} ${PostgreSQL_LIBRARIES}
${YAMLCPP_LIBRARY} ${YAMLCPP_LIBRARY}
${Boost_LIBRARIES}
) )

97
src/commandline.cpp Normal file
View file

@ -0,0 +1,97 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "commandline.hpp"
#include "dindexerConfig.h"
#include <boost/program_options.hpp>
#include <iostream>
#define STRINGIZE_IMPL(s) #s
#define STRINGIZE(s) STRINGIZE_IMPL(s)
namespace po = boost::program_options;
namespace din {
namespace {
const char* const g_version_string =
PROGRAM_NAME " v"
STRINGIZE(VERSION_MAJOR) "."
STRINGIZE(VERSION_MINOR) "."
STRINGIZE(VERSION_PATCH)
#if VERSION_BETA
"b"
#endif
;
} //unnamed namespace
bool parse_commandline (int parArgc, char* parArgv[], po::variables_map& parVarMap) {
po::options_description desc("General");
desc.add_options()
("help,h", "Produces this help message")
("version", "Prints the program's version and quits")
//("dump-raw,D", po::value<std::string>(), "Saves the retrieved html to the named file; use - for stdout")
;
po::options_description set_options("Set options");
set_options.add_options()
("setname,n", po::value<std::string>()->default_value("New set"), "Name to be given to the new set being scanned.")
;
po::options_description positional_options("Positional options");
positional_options.add_options()
("search-path", po::value<std::string>(), "Search path")
;
po::options_description all("Available options");
all.add(desc).add(positional_options).add(set_options);
po::positional_options_description pd;
pd.add("search-path", 1);//.add("xpath", 1);
try {
po::store(po::command_line_parser(parArgc, parArgv).options(all).positional(pd).run(), parVarMap);
}
catch (const po::unknown_option& err) {
throw std::invalid_argument(err.what());
}
po::notify(parVarMap);
if (parVarMap.count("help")) {
#if !defined(NDEBUG)
std::cout << "*******************\n";
std::cout << "*** DEBUG BUILD ***\n";
std::cout << "*******************\n";
std::cout << '\n';
#endif
po::options_description visible("Available options");
visible.add(desc).add(set_options);
std::cout << PROGRAM_NAME << " Copyright (C) 2015 Michele Santullo\n";
std::cout << "This program comes with ABSOLUTELY NO WARRANTY.\n"; //for details type `show w'.
std::cout << "This is free software, and you are welcome to\n";
std::cout << "redistribute it under certain conditions.\n"; //type `show c' for details.
std::cout << '\n';
std::cout << "Usage: " << PROGRAM_NAME << " [options...] <search-path>\n";
std::cout << visible;
return true;
}
else if (parVarMap.count("version")) {
std::cout << g_version_string << '\n';
return true;
}
if (parVarMap.count("search-path") == 0) {
throw std::invalid_argument("No search path specified");
}
return false;
}
} //namespace din

27
src/commandline.hpp Normal file
View file

@ -0,0 +1,27 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id1B7A42F6E46547A6AB0F914E2A91399F
#define id1B7A42F6E46547A6AB0F914E2A91399F
#include <boost/program_options/variables_map.hpp>
namespace din {
bool parse_commandline ( int parArgc, char* parArgv[], boost::program_options::variables_map& parVarMap );
} //namespace din
#endif

View file

@ -25,9 +25,17 @@
namespace din { namespace din {
namespace { namespace {
const std::size_t g_batch_size = 100; const std::size_t g_batch_size = 100;
std::string make_set_insert_query (pq::Connection& parConn, const std::string& parSetName) {
std::ostringstream oss;
oss << "INSERT INTO \"sets\" (\"desc\") VALUES ("
<< parConn.escaped_literal(parSetName)
<< ");";
return oss.str();
}
} //unnamed namespace } //unnamed namespace
void write_to_db (const DinDBSettings& parDB, const std::vector<FileRecordData>& parData) { void write_to_db (const DinDBSettings& parDB, const std::vector<FileRecordData>& parData, const std::string& parSetName) {
if (parData.empty()) { if (parData.empty()) {
return; return;
} }
@ -36,7 +44,7 @@ namespace din {
conn.connect(); conn.connect();
conn.query_void("BEGIN;"); conn.query_void("BEGIN;");
conn.query_void("INSERT INTO \"sets\" (\"desc\") VALUES ('test group');"); conn.query_void(make_set_insert_query(conn, parSetName));
//TODO: use COPY instead of INSERT INTO //TODO: use COPY instead of INSERT INTO
for (std::size_t z = 0; z < parData.size(); z += g_batch_size) { for (std::size_t z = 0; z < parData.size(); z += g_batch_size) {
std::ostringstream query; std::ostringstream query;
@ -46,7 +54,7 @@ namespace din {
for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) { for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) {
const auto& itm = parData[i]; const auto& itm = parData[i];
query << comma; query << comma;
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "'," query << '(' << conn.escaped_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ',' << itm.level << ','
<< "currval('\"sets_id_seq\"')" << ',' << "currval('\"sets_id_seq\"')" << ','
<< (itm.is_directory ? "true" : "false") << ',' << (itm.is_directory ? "true" : "false") << ','

View file

@ -34,7 +34,7 @@ namespace din {
const bool is_symlink; const bool is_symlink;
}; };
void write_to_db ( const DinDBSettings& parDB, const std::vector<FileRecordData>& parData ); void write_to_db ( const DinDBSettings& parDB, const std::vector<FileRecordData>& parData, const std::string& parSetName );
} //namespace din } //namespace din
#endif #endif

27
src/dindexerConfig.h.in Normal file
View file

@ -0,0 +1,27 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id48D6E1D45238460F99C2BCBFDE920791
#define id48D6E1D45238460F99C2BCBFDE920791
#define PROGRAM_NAME "@PROJECT_NAME@"
#define VERSION_MAJOR @PROJECT_VERSION_MAJOR@
#define VERSION_MINOR @PROJECT_VERSION_MINOR@
#define VERSION_BETA @PROJECT_VERSION_BETA@
#define VERSION_PATCH @PROJECT_VERSION_PATCH@
#endif

View file

@ -126,9 +126,9 @@ namespace fastf {
} }
} //unnamed namespace } //unnamed namespace
FileSearcher::FileSearcher (const char* parBaseDir) : FileSearcher::FileSearcher (boost::string_ref parBaseDir) :
callback_(&print_def_callback), callback_(&print_def_callback),
baseDir_(parBaseDir), baseDir_(parBaseDir.begin(), parBaseDir.end()),
followSymlinks_(false), followSymlinks_(false),
remainInFilesystem_(true) remainInFilesystem_(true)
{ {
@ -147,7 +147,7 @@ namespace fastf {
g_searchOptions.extensions = &parExtensions; g_searchOptions.extensions = &parExtensions;
g_searchOptions.ignorePaths = &parIgnorePaths; g_searchOptions.ignorePaths = &parIgnorePaths;
g_searchOptions.callback = &callback_; g_searchOptions.callback = &callback_;
nftw(baseDir_, &PrintName, 15, g_searchOptions.search_flags); nftw(baseDir_.c_str(), &PrintName, 15, g_searchOptions.search_flags);
} }
void FileSearcher::SetCallback (CallbackType parCallback) { void FileSearcher::SetCallback (CallbackType parCallback) {

View file

@ -20,6 +20,7 @@
#include <vector> #include <vector>
#include <functional> #include <functional>
#include <boost/utility/string_ref.hpp>
namespace fastf { namespace fastf {
struct StringWithLength { struct StringWithLength {
@ -38,7 +39,7 @@ namespace fastf {
typedef std::vector<StringWithLength> ConstCharVecType; typedef std::vector<StringWithLength> ConstCharVecType;
typedef std::function<bool(const char*, int, bool, bool)> CallbackType; typedef std::function<bool(const char*, int, bool, bool)> CallbackType;
explicit FileSearcher ( const char* parBaseDir ); explicit FileSearcher ( boost::string_ref parBaseDir );
~FileSearcher ( void ) noexcept; ~FileSearcher ( void ) noexcept;
void Search ( const ConstCharVecType& parExtensions, const ConstCharVecType& parIgnorePaths ); void Search ( const ConstCharVecType& parExtensions, const ConstCharVecType& parIgnorePaths );
@ -51,7 +52,7 @@ namespace fastf {
private: private:
CallbackType callback_; CallbackType callback_;
const char* const baseDir_; const std::string baseDir_;
bool followSymlinks_; bool followSymlinks_;
bool remainInFilesystem_; bool remainInFilesystem_;
}; };

View file

@ -233,8 +233,10 @@ namespace din {
assert(not (1 == itm.hash.part_a and 1 == itm.hash.part_b and 1 == itm.hash.part_c)); assert(not (1 == itm.hash.part_a and 1 == itm.hash.part_b and 1 == itm.hash.part_c));
} }
#endif #endif
}
{ void Indexer::add_to_db (const std::string& parSetName) const {
PathName base_path(m_local_data->paths.front().path);
std::vector<FileRecordData> data; std::vector<FileRecordData> data;
data.reserve(m_local_data->paths.size()); data.reserve(m_local_data->paths.size());
for (const auto& itm : m_local_data->paths) { for (const auto& itm : m_local_data->paths) {
@ -247,8 +249,7 @@ namespace din {
itm.is_symlink itm.is_symlink
}); });
} }
write_to_db(m_local_data->db_settings, data); write_to_db(m_local_data->db_settings, data, parSetName);
}
} }
bool Indexer::add_path (const char* parPath, int parLevel, bool parIsDir, bool parIsSymLink) { bool Indexer::add_path (const char* parPath, int parLevel, bool parIsDir, bool parIsSymLink) {
@ -279,4 +280,8 @@ namespace din {
} }
} }
#endif #endif
bool Indexer::empty() const {
return m_local_data->paths.size() < 2;
}
} //namespace din } //namespace din

View file

@ -38,6 +38,8 @@ namespace din {
std::size_t total_items ( void ) const; std::size_t total_items ( void ) const;
std::size_t processed_items ( void ) const; std::size_t processed_items ( void ) const;
void calculate_hash ( void ); void calculate_hash ( void );
void add_to_db ( const std::string& parSetName ) const;
bool empty ( void ) const;
private: private:
struct LocalData; struct LocalData;

View file

@ -20,19 +20,27 @@
#include "filesearcher.hpp" #include "filesearcher.hpp"
#include "indexer.hpp" #include "indexer.hpp"
#include "settings.hpp" #include "settings.hpp"
#include "commandline.hpp"
int main (int parArgc, char* parArgv[]) { int main (int parArgc, char* parArgv[]) {
using std::placeholders::_1; using std::placeholders::_1;
using std::placeholders::_2; using std::placeholders::_2;
using std::placeholders::_3; using std::placeholders::_3;
using std::placeholders::_4; using std::placeholders::_4;
std::cout << "argc: " << parArgc << '\n'; using boost::program_options::variables_map;
for (int z = 0; z < parArgc; ++z) {
std::cout << "argv[" << z << "]: " << parArgv[z] << '\n'; variables_map vm;
} try {
std::cout << std::endl; if (din::parse_commandline(parArgc, parArgv, vm)) {
return 0;
}
}
catch (const std::invalid_argument& err) {
std::cerr << err.what() << "\nUse --help for help" << std::endl;
return 2;
}
const std::string search_path(vm["search-path"].as<std::string>());
fastf::FileSearcher searcher("/home/michele/dev/code/cpp/dindexer/test");
din::DinDBSettings settings; din::DinDBSettings settings;
{ {
const bool loaded = din::load_settings("dindexerrc.yml", settings); const bool loaded = din::load_settings("dindexerrc.yml", settings);
@ -43,11 +51,19 @@ int main (int parArgc, char* parArgv[]) {
} }
din::Indexer indexer(settings); din::Indexer indexer(settings);
fastf::FileSearcher searcher(search_path);
fastf::FileSearcher::ConstCharVecType ext, ignore; fastf::FileSearcher::ConstCharVecType ext, ignore;
searcher.SetFollowSymlinks(true); searcher.SetFollowSymlinks(true);
searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&din::Indexer::add_path, &indexer, _1, _2, _3, _4))); searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&din::Indexer::add_path, &indexer, _1, _2, _3, _4)));
searcher.Search(ext, ignore); searcher.Search(ext, ignore);
if (indexer.empty()) {
std::cerr << "Nothing found at the given location, quitting\n";
return 1;
}
else {
indexer.calculate_hash(); indexer.calculate_hash();
indexer.add_to_db(vm["setname"].as<std::string>());
}
return 0; return 0;
} }

View file

@ -124,7 +124,7 @@ namespace pq {
} }
} }
std::string Connection::escape_literal (const std::string& parString) { std::string Connection::escaped_literal (const std::string& parString) {
typedef std::unique_ptr<char[], void(*)(void*)> PQArrayType; typedef std::unique_ptr<char[], void(*)(void*)> PQArrayType;
PQArrayType clean_str(PQescapeLiteral(m_localData->connection, parString.c_str(), parString.size()), &PQfreemem); PQArrayType clean_str(PQescapeLiteral(m_localData->connection, parString.c_str(), parString.size()), &PQfreemem);

View file

@ -36,7 +36,7 @@ namespace pq {
void query_void ( const std::string& parQuery ); void query_void ( const std::string& parQuery );
ResultSet query ( const std::string& parQuery ); ResultSet query ( const std::string& parQuery );
std::string escape_literal ( const std::string& parString ); std::string escaped_literal ( const std::string& parString );
private: private:
struct LocalData; struct LocalData;