King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2024-11-25 00:53:43 +00:00
Code Issues Releases Wiki Activity

Allow passing set name and search path as command line parameters.

Browse source
This commit is contained in:
King_DuckZ 2015-11-11 19:24:10 +00:00
parent e9a7e756e1
commit 585c7f45b7
13 changed files with 228 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
CMakeLists.txt
View file

@ -1,13 +1,19 @@
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(dindexer VERSION 0.1 LANGUAGES CXX C)
project(dindexer VERSION 0.1.0 LANGUAGES CXX C)
list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules)
set(PROJECT_VERSION_BETA "1")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++11")
find_library(Boost REQUIRED 1.53.0)
find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options)
find_package(PostgreSQL REQUIRED)
find_package(YamlCpp REQUIRED 0.5.1)
find_package(YamlCpp 0.5.1 REQUIRED)
configure_file(
"${PROJECT_SOURCE_DIR}/src/${PROJECT_NAME}Config.h.in"
"${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.h"
)
add_executable(${PROJECT_NAME}
src/main.cpp
@ -21,10 +27,11 @@ add_executable(${PROJECT_NAME}
src/pq/resultset.cpp
src/dbbackend.cpp
src/settings.cpp
src/commandline.cpp
)
target_include_directories(${PROJECT_NAME} SYSTEM
INTERFACE ${Boost_INCLUDE_DIR}
INTERFACE ${Boost_INCLUDE_DIRS}
PRIVATE ${PostgreSQL_INCLUDE_DIRS}
PRIVATE ${YAMLCPP_INCLUDE_DIR}
)
@ -32,9 +39,11 @@ target_include_directories(${PROJECT_NAME} SYSTEM
target_include_directories(${PROJECT_NAME}
PRIVATE src
PRIVATE include
PRIVATE ${PROJECT_BINARY_DIR}
)
target_link_libraries(${PROJECT_NAME}
${PostgreSQL_LIBRARIES}
${YAMLCPP_LIBRARY}
${Boost_LIBRARIES}
)

97
src/commandline.cpp Normal file
View file

@ -0,0 +1,97 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "commandline.hpp"
#include "dindexerConfig.h"
#include <boost/program_options.hpp>
#include <iostream>
#define STRINGIZE_IMPL(s) #s
#define STRINGIZE(s) STRINGIZE_IMPL(s)
namespace po = boost::program_options;
namespace din {
namespace {
const char* const g_version_string =
PROGRAM_NAME " v"
STRINGIZE(VERSION_MAJOR) "."
STRINGIZE(VERSION_MINOR) "."
STRINGIZE(VERSION_PATCH)
#if VERSION_BETA
"b"
#endif
;
} //unnamed namespace
bool parse_commandline (int parArgc, char* parArgv[], po::variables_map& parVarMap) {
po::options_description desc("General");
desc.add_options()
("help,h", "Produces this help message")
("version", "Prints the program's version and quits")
//("dump-raw,D", po::value<std::string>(), "Saves the retrieved html to the named file; use - for stdout")
;
po::options_description set_options("Set options");
set_options.add_options()
("setname,n", po::value<std::string>()->default_value("New set"), "Name to be given to the new set being scanned.")
;
po::options_description positional_options("Positional options");
positional_options.add_options()
("search-path", po::value<std::string>(), "Search path")
;
po::options_description all("Available options");
all.add(desc).add(positional_options).add(set_options);
po::positional_options_description pd;
pd.add("search-path", 1);//.add("xpath", 1);
try {
po::store(po::command_line_parser(parArgc, parArgv).options(all).positional(pd).run(), parVarMap);
}
catch (const po::unknown_option& err) {
throw std::invalid_argument(err.what());
}
po::notify(parVarMap);
if (parVarMap.count("help")) {
#if !defined(NDEBUG)
std::cout << "*******************\n";
std::cout << "*** DEBUG BUILD ***\n";
std::cout << "*******************\n";
std::cout << '\n';
#endif
po::options_description visible("Available options");
visible.add(desc).add(set_options);
std::cout << PROGRAM_NAME << " Copyright (C) 2015 Michele Santullo\n";
std::cout << "This program comes with ABSOLUTELY NO WARRANTY.\n"; //for details type `show w'.
std::cout << "This is free software, and you are welcome to\n";
std::cout << "redistribute it under certain conditions.\n"; //type `show c' for details.
std::cout << '\n';
std::cout << "Usage: " << PROGRAM_NAME << " [options...] <search-path>\n";
std::cout << visible;
return true;
}
else if (parVarMap.count("version")) {
std::cout << g_version_string << '\n';
return true;
}
if (parVarMap.count("search-path") == 0) {
throw std::invalid_argument("No search path specified");
}
return false;
}
} //namespace din

27
src/commandline.hpp Normal file
View file

@ -0,0 +1,27 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id1B7A42F6E46547A6AB0F914E2A91399F
#define id1B7A42F6E46547A6AB0F914E2A91399F
#include <boost/program_options/variables_map.hpp>
namespace din {
bool parse_commandline ( int parArgc, char* parArgv[], boost::program_options::variables_map& parVarMap );
} //namespace din
#endif

14
src/dbbackend.cpp
View file

@ -25,9 +25,17 @@
namespace din {
namespace {
const std::size_t g_batch_size = 100;
std::string make_set_insert_query (pq::Connection& parConn, const std::string& parSetName) {
std::ostringstream oss;
oss << "INSERT INTO \"sets\" (\"desc\") VALUES ("
<< parConn.escaped_literal(parSetName)
<< ");";
return oss.str();
}
} //unnamed namespace
void write_to_db (const DinDBSettings& parDB, const std::vector<FileRecordData>& parData) {
void write_to_db (const DinDBSettings& parDB, const std::vector<FileRecordData>& parData, const std::string& parSetName) {
if (parData.empty()) {
return;
}
@ -36,7 +44,7 @@ namespace din {
conn.connect();
conn.query_void("BEGIN;");
conn.query_void("INSERT INTO \"sets\" (\"desc\") VALUES ('test group');");
conn.query_void(make_set_insert_query(conn, parSetName));
//TODO: use COPY instead of INSERT INTO
for (std::size_t z = 0; z < parData.size(); z += g_batch_size) {
std::ostringstream query;
@ -46,7 +54,7 @@ namespace din {
for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) {
const auto& itm = parData[i];
query << comma;
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "',"
query << '(' << conn.escaped_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ','
<< "currval('\"sets_id_seq\"')" << ','
<< (itm.is_directory ? "true" : "false") << ','

2
src/dbbackend.hpp
View file

@ -34,7 +34,7 @@ namespace din {
const bool is_symlink;
};
void write_to_db ( const DinDBSettings& parDB, const std::vector<FileRecordData>& parData );
void write_to_db ( const DinDBSettings& parDB, const std::vector<FileRecordData>& parData, const std::string& parSetName );
} //namespace din
#endif

27
src/dindexerConfig.h.in Normal file
View file

@ -0,0 +1,27 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id48D6E1D45238460F99C2BCBFDE920791
#define id48D6E1D45238460F99C2BCBFDE920791
#define PROGRAM_NAME "@PROJECT_NAME@"
#define VERSION_MAJOR @PROJECT_VERSION_MAJOR@
#define VERSION_MINOR @PROJECT_VERSION_MINOR@
#define VERSION_BETA @PROJECT_VERSION_BETA@
#define VERSION_PATCH @PROJECT_VERSION_PATCH@
#endif

6
src/filesearcher.cpp
View file

@ -126,9 +126,9 @@ namespace fastf {
}
} //unnamed namespace
FileSearcher::FileSearcher (const char* parBaseDir) :
FileSearcher::FileSearcher (boost::string_ref parBaseDir) :
callback_(&print_def_callback),
baseDir_(parBaseDir),
baseDir_(parBaseDir.begin(), parBaseDir.end()),
followSymlinks_(false),
remainInFilesystem_(true)
{
@ -147,7 +147,7 @@ namespace fastf {
g_searchOptions.extensions = &parExtensions;
g_searchOptions.ignorePaths = &parIgnorePaths;
g_searchOptions.callback = &callback_;
nftw(baseDir_, &PrintName, 15, g_searchOptions.search_flags);
nftw(baseDir_.c_str(), &PrintName, 15, g_searchOptions.search_flags);
}
void FileSearcher::SetCallback (CallbackType parCallback) {

5
src/filesearcher.hpp
View file

@ -20,6 +20,7 @@
#include <vector>
#include <functional>
#include <boost/utility/string_ref.hpp>
namespace fastf {
struct StringWithLength {
@ -38,7 +39,7 @@ namespace fastf {
typedef std::vector<StringWithLength> ConstCharVecType;
typedef std::function<bool(const char*, int, bool, bool)> CallbackType;
explicit FileSearcher ( const char* parBaseDir );
explicit FileSearcher ( boost::string_ref parBaseDir );
~FileSearcher ( void ) noexcept;
void Search ( const ConstCharVecType& parExtensions, const ConstCharVecType& parIgnorePaths );
@ -51,7 +52,7 @@ namespace fastf {
private:
CallbackType callback_;
const char* const baseDir_;
const std::string baseDir_;
bool followSymlinks_;
bool remainInFilesystem_;
};

33
src/indexer.cpp
View file

@ -233,22 +233,23 @@ namespace din {
assert(not (1 == itm.hash.part_a and 1 == itm.hash.part_b and 1 == itm.hash.part_c));
}
#endif
}
{
std::vector<FileRecordData> data;
data.reserve(m_local_data->paths.size());
for (const auto& itm : m_local_data->paths) {
data.push_back(FileRecordData {
make_relative_path(base_path, PathName(itm.path)).path(),
tiger_to_string(itm.hash),
itm.level,
itm.file_size,
itm.is_dir,
itm.is_symlink
});
}
write_to_db(m_local_data->db_settings, data);
void Indexer::add_to_db (const std::string& parSetName) const {
PathName base_path(m_local_data->paths.front().path);
std::vector<FileRecordData> data;
data.reserve(m_local_data->paths.size());
for (const auto& itm : m_local_data->paths) {
data.push_back(FileRecordData {
make_relative_path(base_path, PathName(itm.path)).path(),
tiger_to_string(itm.hash),
itm.level,
itm.file_size,
itm.is_dir,
itm.is_symlink
});
}
write_to_db(m_local_data->db_settings, data, parSetName);
}
bool Indexer::add_path (const char* parPath, int parLevel, bool parIsDir, bool parIsSymLink) {
@ -279,4 +280,8 @@ namespace din {
}
}
#endif
bool Indexer::empty() const {
return m_local_data->paths.size() < 2;
}
} //namespace din

2
src/indexer.hpp
View file

@ -38,6 +38,8 @@ namespace din {
std::size_t total_items ( void ) const;
std::size_t processed_items ( void ) const;
void calculate_hash ( void );
void add_to_db ( const std::string& parSetName ) const;
bool empty ( void ) const;
private:
struct LocalData;

30
src/main.cpp
View file

@ -20,19 +20,27 @@
#include "filesearcher.hpp"
#include "indexer.hpp"
#include "settings.hpp"
#include "commandline.hpp"
int main (int parArgc, char* parArgv[]) {
using std::placeholders::_1;
using std::placeholders::_2;
using std::placeholders::_3;
using std::placeholders::_4;
std::cout << "argc: " << parArgc << '\n';
for (int z = 0; z < parArgc; ++z) {
std::cout << "argv[" << z << "]: " << parArgv[z] << '\n';
}
std::cout << std::endl;
using boost::program_options::variables_map;
variables_map vm;
try {
if (din::parse_commandline(parArgc, parArgv, vm)) {
return 0;
}
}
catch (const std::invalid_argument& err) {
std::cerr << err.what() << "\nUse --help for help" << std::endl;
return 2;
}
const std::string search_path(vm["search-path"].as<std::string>());
fastf::FileSearcher searcher("/home/michele/dev/code/cpp/dindexer/test");
din::DinDBSettings settings;
{
const bool loaded = din::load_settings("dindexerrc.yml", settings);
@ -43,11 +51,19 @@ int main (int parArgc, char* parArgv[]) {
}
din::Indexer indexer(settings);
fastf::FileSearcher searcher(search_path);
fastf::FileSearcher::ConstCharVecType ext, ignore;
searcher.SetFollowSymlinks(true);
searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&din::Indexer::add_path, &indexer, _1, _2, _3, _4)));
searcher.Search(ext, ignore);
indexer.calculate_hash();
if (indexer.empty()) {
std::cerr << "Nothing found at the given location, quitting\n";
return 1;
}
else {
indexer.calculate_hash();
indexer.add_to_db(vm["setname"].as<std::string>());
}
return 0;
}

2
src/pq/connection.cpp
View file

@ -124,7 +124,7 @@ namespace pq {
}
}
std::string Connection::escape_literal (const std::string& parString) {
std::string Connection::escaped_literal (const std::string& parString) {
typedef std::unique_ptr<char[], void(*)(void*)> PQArrayType;
PQArrayType clean_str(PQescapeLiteral(m_localData->connection, parString.c_str(), parString.size()), &PQfreemem);

2
src/pq/connection.hpp
View file

@ -36,7 +36,7 @@ namespace pq {
void query_void ( const std::string& parQuery );
ResultSet query ( const std::string& parQuery );
std::string escape_literal ( const std::string& parString );
std::string escaped_literal ( const std::string& parString );
private:
struct LocalData;