1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-02-19 12:04:54 +00:00

Send data to the database.

This commit is contained in:
King_DuckZ 2015-11-11 02:01:37 +00:00
parent a00d30b0ee
commit bd9ce0ef54
12 changed files with 674 additions and 1 deletions

View file

@ -5,6 +5,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++11")
find_library (Boost REQUIRED 1.53.0)
find_package(PostgreSQL REQUIRED)
add_executable(${PROJECT_NAME}
src/main.cpp
@ -13,8 +14,22 @@ add_executable(${PROJECT_NAME}
src/indexer.cpp
src/tiger.c
src/tiger.cpp
src/pq/connection.cpp
src/pq/databaseexception.cpp
src/pq/resultset.cpp
src/dbbackend.cpp
)
target_include_directories(${PROJECT_NAME} SYSTEM
INTERFACE ${Boost_INCLUDE_DIR}
PRIVATE ${PostgreSQL_INCLUDE_DIRS}
)
target_include_directories(${PROJECT_NAME}
PRIVATE src
PRIVATE include
)
target_link_libraries(${PROJECT_NAME}
${PostgreSQL_LIBRARIES}
)

View file

@ -0,0 +1,39 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id031383D06E3E4D6FBD77625350CDF083
#define id031383D06E3E4D6FBD77625350CDF083
#include <stdexcept>
#include <string>
#include <cstddef>
namespace pq {
class DatabaseException : public std::runtime_error {
public:
DatabaseException ( void ) = delete;
~DatabaseException ( void ) noexcept = default;
DatabaseException ( std::string&& parMsg, std::string&& parErr, const std::string& parFile, std::size_t parLine );
const std::string& error_message ( void ) const noexcept { return m_error; }
private:
const std::string m_error;
};
} //namespace pq
#endif

84
include/pq/resultset.hpp Normal file
View file

@ -0,0 +1,84 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef idA54B379097864A0BA1B7F9C42B74EAAD
#define idA54B379097864A0BA1B7F9C42B74EAAD
#include <cstddef>
#include <string>
#include <memory>
#include <boost/iterator/counting_iterator.hpp>
#include <boost/iterator/transform_iterator.hpp>
#include <boost/function.hpp>
namespace pq {
class Connection;
struct ResultInfo;
class Row {
public:
typedef boost::transform_iterator<boost::function<std::string(std::size_t)>, boost::counting_iterator<std::size_t>> const_iterator;
Row ( const ResultInfo& parResult, std::size_t parRow );
Row ( const Row& ) = delete;
Row ( Row&& parOther );
~Row ( void ) noexcept;
bool empty ( void ) const;
std::size_t size ( void ) const;
std::string operator[] ( const std::string& parIndex ) const;
std::string operator[] ( std::size_t parIndex ) const;
Row& operator= ( const Row& ) = delete;
Row& operator= ( Row&& ) = delete;
const_iterator begin ( void ) const;
const_iterator end ( void ) const;
const_iterator cbegin ( void ) const { return begin(); }
const_iterator cend ( void ) const { return end(); }
private:
struct LocalData;
std::unique_ptr<LocalData> m_localData;
};
class ResultSet {
public:
typedef boost::transform_iterator<boost::function<Row(std::size_t)>, boost::counting_iterator<std::size_t>> const_iterator;
explicit ResultSet ( ResultInfo& parResult );
ResultSet ( const ResultSet& ) = delete;
ResultSet ( ResultSet&& parOther );
~ResultSet ( void ) noexcept;
bool empty ( void ) const;
std::size_t size ( void ) const;
Row operator[] ( const std::size_t parIndex ) const;
const_iterator begin ( void ) const;
const_iterator end ( void ) const;
const_iterator cbegin ( void ) const { return begin(); }
const_iterator cend ( void ) const { return end(); }
private:
struct LocalData;
std::unique_ptr<LocalData> m_localData;
};
} //namespace pq
#endif

49
src/dbbackend.cpp Normal file
View file

@ -0,0 +1,49 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "dbbackend.hpp"
#include "pq/connection.hpp"
#include <string>
#include <sstream>
namespace din {
void write_to_db (const std::vector<FileRecordData>& parData) {
if (parData.empty()) {
return;
}
std::ostringstream query;
query << "BEGIN;\n";
query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES ";
const char* comma = "";
for (const auto& itm : parData) {
query << comma;
query << "('" << itm.path << "','" << itm.hash << "'," << itm.level << ','
<< 10 << ',' << (itm.is_directory ? "true" : "false") << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')'
;
comma = ",";
}
query << ';';
query << "\nCOMMIT;";
pq::Connection conn("michele", "password", "dindexer", "100.200.100.200", 5432);
conn.connect();
conn.query_void(query.str());
}
} //namespace din

38
src/dbbackend.hpp Normal file
View file

@ -0,0 +1,38 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id842AF56BD80A4CF59957451DF9082AA2
#define id842AF56BD80A4CF59957451DF9082AA2
#include <boost/utility/string_ref.hpp>
#include <vector>
#include <utility>
namespace din {
struct FileRecordData {
const boost::string_ref path;
const std::string hash;
const uint16_t level;
const std::size_t size;
const bool is_directory;
const bool is_symlink;
};
void write_to_db ( const std::vector<FileRecordData>& parData );
} //namespace din
#endif

View file

@ -18,6 +18,7 @@
#include "indexer.hpp"
#include "pathname.hpp"
#include "tiger.hpp"
#include "dbbackend.hpp"
#include <algorithm>
#include <vector>
#include <string>
@ -207,6 +208,22 @@ namespace din {
assert(not (1 == itm.hash.part_a and 1 == itm.hash.part_b and 1 == itm.hash.part_c));
}
#endif
{
std::vector<FileRecordData> data;
data.reserve(m_local_data->paths.size());
for (const auto& itm : m_local_data->paths) {
data.push_back(FileRecordData {
boost::string_ref(itm.path),
tiger_to_string(itm.hash),
itm.level,
0,
itm.is_dir,
itm.is_symlink
});
}
write_to_db(data);
}
}
bool Indexer::add_path (const char* parPath, int parLevel, bool parIsDir, bool parIsSymLink) {

126
src/pq/connection.cpp Normal file
View file

@ -0,0 +1,126 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "pq/connection.hpp"
#include "pq/databaseexception.hpp"
#include "pq/resultinfo.hpp"
#include <libpq-fe.h>
#include <algorithm>
#include <ciso646>
#include <memory>
#include <boost/lexical_cast.hpp>
#include <sstream>
namespace pq {
struct Connection::LocalData {
PGconn* connection;
};
Connection::Connection (std::string&& parUsername, std::string&& parPasswd, std::string&& parDatabase, std::string&& parAddress, uint16_t parPort) :
m_username(std::move(parUsername)),
m_passwd(std::move(parPasswd)),
m_database(std::move(parDatabase)),
m_address(std::move(parAddress)),
m_port(parPort),
m_localData(new LocalData)
{
m_localData->connection = nullptr;
}
Connection::~Connection() noexcept {
disconnect();
}
bool Connection::is_connected() const noexcept {
return m_localData->connection != nullptr;
}
void Connection::connect() {
assert(not is_connected());
std::unique_ptr<const char*[]> names(new const char*[6]);
names[0] = "host";
names[1] = "port";
names[2] = "dbname";
names[3] = "user";
names[4] = "password";
names[5] = nullptr;
std::unique_ptr<const char*[]> keywords(new const char*[6]);
const std::string port(boost::lexical_cast<std::string>(m_port));
keywords[0] = m_address.c_str();
keywords[1] = port.c_str();
keywords[2] = m_database.c_str();
keywords[3] = m_username.c_str();
keywords[4] = m_passwd.c_str();
keywords[5] = nullptr;
m_localData->connection = PQconnectdbParams(names.get(), keywords.get(), 0);
if (not m_localData->connection)
throw DatabaseException("", "Error allocatinng connection object", __FILE__, __LINE__);
if (PQstatus(m_localData->connection) != CONNECTION_OK) {
std::string err = error_message();
disconnect();
std::ostringstream oss;
oss << "Unable to connect to database " << m_address << ':' << m_port << " as user \"" << m_username << '"';
throw DatabaseException(oss.str(), std::move(err), __FILE__, __LINE__);
}
query_void("SET NAMES 'utf8'");
}
void Connection::disconnect() {
if (is_connected()) {
PQfinish(m_localData->connection);
m_localData->connection = nullptr;
}
}
std::string Connection::error_message() const {
assert(is_connected());
return PQerrorMessage(m_localData->connection);
}
ResultSet Connection::query (const std::string& parQuery) {
//TODO: sanitize input string
auto res = PQexec(m_localData->connection, parQuery.c_str());
if (not res)
throw DatabaseException("Error running query", "Error allocating result object", __FILE__, __LINE__);
const int ress = PQresultStatus(res);
if (ress != PGRES_TUPLES_OK && ress != PGRES_COMMAND_OK) {
PQclear(res);
throw DatabaseException("Error running query", error_message(), __FILE__, __LINE__);
}
ResultInfo info;
info.result = res;
info.column_mappings = nullptr;
return ResultSet(info);
}
void Connection::query_void (const std::string& parQuery) {
//TODO: sanitize input string
auto deleter = [](PGresult* r) { PQclear(r); };
std::unique_ptr<PGresult, decltype(deleter)> res(PQexec(m_localData->connection, parQuery.c_str()), deleter);
if (not res)
throw DatabaseException("Error running query", "Error allocating result object", __FILE__, __LINE__);
const int ress = PQresultStatus(res.get());
if (ress != PGRES_TUPLES_OK && ress != PGRES_COMMAND_OK) {
throw DatabaseException("Error running query", error_message(), __FILE__, __LINE__);
}
}
} //namespace pq

51
src/pq/connection.hpp Normal file
View file

@ -0,0 +1,51 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id68BF7AAC5BD54AF2BBD6EC1318B6687E
#define id68BF7AAC5BD54AF2BBD6EC1318B6687E
#include "pq/resultset.hpp"
#include <string>
#include <cstdint>
#include <memory>
namespace pq {
class Connection {
public:
Connection ( std::string&& parUsername, std::string&& parPasswd, std::string&& parDatabase, std::string&& parAddress, uint16_t parPort );
~Connection ( void ) noexcept;
std::string error_message ( void ) const;
bool is_connected ( void ) const noexcept;
void connect ( void );
void disconnect ( void );
void query_void ( const std::string& parQuery );
ResultSet query ( const std::string& parQuery );
private:
struct LocalData;
const std::string m_username;
const std::string m_passwd;
const std::string m_database;
const std::string m_address;
const uint16_t m_port;
std::unique_ptr<LocalData> m_localData;
};
} //namespace pq
#endif

View file

@ -0,0 +1,39 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/lexical_cast.hpp>
#include "pq/databaseexception.hpp"
namespace pq {
namespace {
std::string& compose_err_msg ( std::string& parMsg, const std::string& parErr, const std::string& parFile, std::size_t parLine) {
parMsg += ": ";
parMsg += parErr;
parMsg += '\n';
parMsg += parFile;
parMsg += ':';
parMsg += boost::lexical_cast<std::string>(parLine);
return parMsg;
}
} //unnamed namespace
DatabaseException::DatabaseException (std::string&& parMsg, std::string&& parErr, const std::string& parFile, std::size_t parLine) :
std::runtime_error(compose_err_msg(parMsg, parErr, parFile, parLine)),
m_error(std::move(parErr))
{
}
} //namespace pq

32
src/pq/resultinfo.hpp Normal file
View file

@ -0,0 +1,32 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef idB756FAED6EE34FB18E302528CEA7C1D7
#define idB756FAED6EE34FB18E302528CEA7C1D7
#include <libpq-fe.h>
#include <map>
#include <string>
namespace pq {
struct ResultInfo {
PGresult* result;
const std::map<std::string, int>* column_mappings;
};
} //namespace pq
#endif

176
src/pq/resultset.cpp Normal file
View file

@ -0,0 +1,176 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "pq/resultset.hpp"
#include "pq/resultinfo.hpp"
#include "pq/databaseexception.hpp"
#include <libpq-fe.h>
#include <map>
#include <cassert>
#include <ciso646>
#include <boost/lexical_cast.hpp>
#include <algorithm>
#include <boost/bind.hpp>
namespace pq {
struct Row::LocalData {
LocalData ( const PGresult* parResult, std::size_t parRow, const std::map<std::string, int>* parCols, std::size_t parColCount ) :
result(parResult),
columns(parCols),
row_index(parRow),
col_count(parColCount)
{
assert((result and columns) or (not (result or columns)));
}
const PGresult* const result;
const std::map<std::string, int>* const columns;
const std::size_t row_index;
const std::size_t col_count;
};
struct ResultSet::LocalData {
explicit LocalData ( PGresult* parResult ) :
result(parResult)
{
}
PGresult* result;
std::map<std::string, int> columns;
};
Row::Row (const ResultInfo& parResult, std::size_t parRow) :
m_localData(new LocalData(parResult.result, parRow, parResult.column_mappings, (parResult.column_mappings ? parResult.column_mappings->size() : 0)))
{
assert(not m_localData->result or static_cast<std::size_t>(std::max(0, PQnfields(m_localData->result))) == m_localData->col_count);
}
Row::Row (Row&& parOther) :
m_localData(std::move(parOther.m_localData))
{
}
//Row::Row (const Row& parOther) :
// m_localData(new LocalData(parOther.m_localData->result, parOther.m_localData->row_index, parOther.m_localData->columns, parOther.m_localData->col_count))
//{
//}
Row::~Row() noexcept {
}
std::string Row::operator[] (const std::string& parIndex) const {
assert(m_localData->columns);
const auto num_index_it = m_localData->columns->find(parIndex);
if (num_index_it == m_localData->columns->end())
throw DatabaseException("Error retrieving column", "Requested column \"" + parIndex + "\" doesn't exist", __FILE__, __LINE__);
const int num_index = num_index_it->second;
return PQgetvalue(m_localData->result, static_cast<int>(m_localData->row_index), num_index);
}
std::size_t Row::size() const {
return m_localData->col_count;
}
bool Row::empty() const {
return size() == 0;
}
std::string Row::operator[] (std::size_t parIndex) const {
if (parIndex >= m_localData->col_count) {
std::ostringstream oss;
oss << "Requested column at index " << parIndex << " is out of range (column count is " << m_localData->col_count << ')';
throw DatabaseException("Error retrieving column", oss.str(), __FILE__, __LINE__);
}
return PQgetvalue(m_localData->result, static_cast<int>(m_localData->row_index), static_cast<int>(parIndex));
}
Row::const_iterator Row::begin() const {
return boost::make_transform_iterator(
boost::counting_iterator<std::size_t>(0),
boost::bind(static_cast<std::string (Row::*)(std::size_t) const>(&Row::operator[]), this, _1)
);
}
Row::const_iterator Row::end() const {
return boost::make_transform_iterator(
boost::counting_iterator<std::size_t>(m_localData->col_count),
boost::bind(static_cast<std::string (Row::*)(std::size_t) const>(&Row::operator[]), this, _1)
);
}
ResultSet::ResultSet (ResultInfo& parResult) :
m_localData(new LocalData(parResult.result))
{
const auto result = m_localData->result;
const int column_count = PQnfields(result);
std::string col_name;
for (int z = 0; z < column_count; ++z) {
m_localData->columns[PQfname(result, z)] = z;
}
}
ResultSet::ResultSet (ResultSet&& parOther) :
m_localData(std::move(parOther.m_localData))
{
}
ResultSet::~ResultSet() noexcept {
if (m_localData->result) {
PQclear(m_localData->result);
m_localData->result = nullptr;
}
}
Row ResultSet::operator[] (std::size_t parIndex) const {
ResultInfo info;
info.result = m_localData->result;
info.column_mappings = &m_localData->columns;
if (parIndex >= size())
throw DatabaseException("Out of range", "Column index " + boost::lexical_cast<std::string>(parIndex) + " is out of range (column count is " + boost::lexical_cast<std::string>(size()) + ')', __FILE__, __LINE__);
return Row(info, parIndex);
}
std::size_t ResultSet::size() const {
assert(m_localData->result);
const auto retval = PQntuples(m_localData->result);
assert(retval >= 0);
return static_cast<std::size_t>(std::max(0, retval));
}
bool ResultSet::empty() const {
return size() == 0;
}
ResultSet::const_iterator ResultSet::begin() const {
return boost::make_transform_iterator(
boost::counting_iterator<std::size_t>(0),
boost::bind(static_cast<Row (ResultSet::*)(std::size_t) const>(&ResultSet::operator[]), this, _1)
);
}
ResultSet::const_iterator ResultSet::end() const {
return boost::make_transform_iterator(
boost::counting_iterator<std::size_t>(size()),
boost::bind(static_cast<Row (ResultSet::*)(std::size_t) const>(&ResultSet::operator[]), this, _1)
);
}
} //namespace pq

View file

@ -23,6 +23,7 @@
#include <algorithm>
#include <utility>
#include <sstream>
#include <iomanip>
#if defined(__SSE2__)
extern "C" void tiger_sse2_chunk ( const char* parStr1, const char* parStr2, uint64_t parLength, uint64_t parRes1[3], uint64_t parRes2[3] );
@ -102,7 +103,13 @@ namespace din {
std::string tiger_to_string (const TigerHash& parHash) {
std::ostringstream oss;
oss << std::hex << swap_long(parHash.part_a) << swap_long(parHash.part_b) << swap_long(parHash.part_c);
oss << std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t))
<< swap_long(parHash.part_a)
<< std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t))
<< swap_long(parHash.part_b)
<< std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t))
<< swap_long(parHash.part_c)
;
return oss.str();
}