diff --git a/CMakeLists.txt b/CMakeLists.txt index e775978..7a35d0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++11") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++11") find_library (Boost REQUIRED 1.53.0) +find_package(PostgreSQL REQUIRED) add_executable(${PROJECT_NAME} src/main.cpp @@ -13,8 +14,22 @@ add_executable(${PROJECT_NAME} src/indexer.cpp src/tiger.c src/tiger.cpp + src/pq/connection.cpp + src/pq/databaseexception.cpp + src/pq/resultset.cpp + src/dbbackend.cpp ) target_include_directories(${PROJECT_NAME} SYSTEM INTERFACE ${Boost_INCLUDE_DIR} + PRIVATE ${PostgreSQL_INCLUDE_DIRS} +) + +target_include_directories(${PROJECT_NAME} + PRIVATE src + PRIVATE include +) + +target_link_libraries(${PROJECT_NAME} + ${PostgreSQL_LIBRARIES} ) diff --git a/include/pq/databaseexception.hpp b/include/pq/databaseexception.hpp new file mode 100644 index 0000000..d3c321e --- /dev/null +++ b/include/pq/databaseexception.hpp @@ -0,0 +1,39 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id031383D06E3E4D6FBD77625350CDF083 +#define id031383D06E3E4D6FBD77625350CDF083 + +#include +#include +#include + +namespace pq { + class DatabaseException : public std::runtime_error { + public: + DatabaseException ( void ) = delete; + ~DatabaseException ( void ) noexcept = default; + DatabaseException ( std::string&& parMsg, std::string&& parErr, const std::string& parFile, std::size_t parLine ); + + const std::string& error_message ( void ) const noexcept { return m_error; } + + private: + const std::string m_error; + }; +} //namespace pq + +#endif diff --git a/include/pq/resultset.hpp b/include/pq/resultset.hpp new file mode 100644 index 0000000..cc287fa --- /dev/null +++ b/include/pq/resultset.hpp @@ -0,0 +1,84 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idA54B379097864A0BA1B7F9C42B74EAAD +#define idA54B379097864A0BA1B7F9C42B74EAAD + +#include +#include +#include +#include +#include +#include + +namespace pq { + class Connection; + struct ResultInfo; + + class Row { + public: + typedef boost::transform_iterator, boost::counting_iterator> const_iterator; + + Row ( const ResultInfo& parResult, std::size_t parRow ); + Row ( const Row& ) = delete; + Row ( Row&& parOther ); + ~Row ( void ) noexcept; + + bool empty ( void ) const; + std::size_t size ( void ) const; + std::string operator[] ( const std::string& parIndex ) const; + std::string operator[] ( std::size_t parIndex ) const; + Row& operator= ( const Row& ) = delete; + Row& operator= ( Row&& ) = delete; + + const_iterator begin ( void ) const; + const_iterator end ( void ) const; + const_iterator cbegin ( void ) const { return begin(); } + const_iterator cend ( void ) const { return end(); } + + private: + struct LocalData; + + std::unique_ptr m_localData; + }; + + class ResultSet { + public: + typedef boost::transform_iterator, boost::counting_iterator> const_iterator; + + explicit ResultSet ( ResultInfo& parResult ); + ResultSet ( const ResultSet& ) = delete; + ResultSet ( ResultSet&& parOther ); + ~ResultSet ( void ) noexcept; + + bool empty ( void ) const; + std::size_t size ( void ) const; + Row operator[] ( const std::size_t parIndex ) const; + + const_iterator begin ( void ) const; + const_iterator end ( void ) const; + const_iterator cbegin ( void ) const { return begin(); } + const_iterator cend ( void ) const { return end(); } + + private: + struct LocalData; + + std::unique_ptr m_localData; + }; +} //namespace pq + +#endif diff --git a/src/dbbackend.cpp b/src/dbbackend.cpp new file mode 100644 index 0000000..6a77cb1 --- /dev/null +++ b/src/dbbackend.cpp @@ -0,0 +1,49 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "dbbackend.hpp" +#include "pq/connection.hpp" +#include +#include + +namespace din { + void write_to_db (const std::vector& parData) { + if (parData.empty()) { + return; + } + + std::ostringstream query; + query << "BEGIN;\n"; + query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES "; + + const char* comma = ""; + for (const auto& itm : parData) { + query << comma; + query << "('" << itm.path << "','" << itm.hash << "'," << itm.level << ',' + << 10 << ',' << (itm.is_directory ? "true" : "false") << ',' + << (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')' + ; + comma = ","; + } + query << ';'; + query << "\nCOMMIT;"; + + pq::Connection conn("michele", "password", "dindexer", "100.200.100.200", 5432); + conn.connect(); + conn.query_void(query.str()); + } +} //namespace din diff --git a/src/dbbackend.hpp b/src/dbbackend.hpp new file mode 100644 index 0000000..548e55c --- /dev/null +++ b/src/dbbackend.hpp @@ -0,0 +1,38 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id842AF56BD80A4CF59957451DF9082AA2 +#define id842AF56BD80A4CF59957451DF9082AA2 + +#include +#include +#include + +namespace din { + struct FileRecordData { + const boost::string_ref path; + const std::string hash; + const uint16_t level; + const std::size_t size; + const bool is_directory; + const bool is_symlink; + }; + + void write_to_db ( const std::vector& parData ); +} //namespace din + +#endif diff --git a/src/indexer.cpp b/src/indexer.cpp index ddf9634..149ec35 100644 --- a/src/indexer.cpp +++ b/src/indexer.cpp @@ -18,6 +18,7 @@ #include "indexer.hpp" #include "pathname.hpp" #include "tiger.hpp" +#include "dbbackend.hpp" #include #include #include @@ -207,6 +208,22 @@ namespace din { assert(not (1 == itm.hash.part_a and 1 == itm.hash.part_b and 1 == itm.hash.part_c)); } #endif + + { + std::vector data; + data.reserve(m_local_data->paths.size()); + for (const auto& itm : m_local_data->paths) { + data.push_back(FileRecordData { + boost::string_ref(itm.path), + tiger_to_string(itm.hash), + itm.level, + 0, + itm.is_dir, + itm.is_symlink + }); + } + write_to_db(data); + } } bool Indexer::add_path (const char* parPath, int parLevel, bool parIsDir, bool parIsSymLink) { diff --git a/src/pq/connection.cpp b/src/pq/connection.cpp new file mode 100644 index 0000000..f1b9b57 --- /dev/null +++ b/src/pq/connection.cpp @@ -0,0 +1,126 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "pq/connection.hpp" +#include "pq/databaseexception.hpp" +#include "pq/resultinfo.hpp" +#include +#include +#include +#include +#include +#include + +namespace pq { + struct Connection::LocalData { + PGconn* connection; + }; + + Connection::Connection (std::string&& parUsername, std::string&& parPasswd, std::string&& parDatabase, std::string&& parAddress, uint16_t parPort) : + m_username(std::move(parUsername)), + m_passwd(std::move(parPasswd)), + m_database(std::move(parDatabase)), + m_address(std::move(parAddress)), + m_port(parPort), + m_localData(new LocalData) + { + m_localData->connection = nullptr; + } + + Connection::~Connection() noexcept { + disconnect(); + } + + bool Connection::is_connected() const noexcept { + return m_localData->connection != nullptr; + } + + void Connection::connect() { + assert(not is_connected()); + + std::unique_ptr names(new const char*[6]); + names[0] = "host"; + names[1] = "port"; + names[2] = "dbname"; + names[3] = "user"; + names[4] = "password"; + names[5] = nullptr; + + std::unique_ptr keywords(new const char*[6]); + const std::string port(boost::lexical_cast(m_port)); + keywords[0] = m_address.c_str(); + keywords[1] = port.c_str(); + keywords[2] = m_database.c_str(); + keywords[3] = m_username.c_str(); + keywords[4] = m_passwd.c_str(); + keywords[5] = nullptr; + + m_localData->connection = PQconnectdbParams(names.get(), keywords.get(), 0); + if (not m_localData->connection) + throw DatabaseException("", "Error allocatinng connection object", __FILE__, __LINE__); + if (PQstatus(m_localData->connection) != CONNECTION_OK) { + std::string err = error_message(); + disconnect(); + + std::ostringstream oss; + oss << "Unable to connect to database " << m_address << ':' << m_port << " as user \"" << m_username << '"'; + throw DatabaseException(oss.str(), std::move(err), __FILE__, __LINE__); + } + query_void("SET NAMES 'utf8'"); + } + + void Connection::disconnect() { + if (is_connected()) { + PQfinish(m_localData->connection); + m_localData->connection = nullptr; + } + } + + std::string Connection::error_message() const { + assert(is_connected()); + return PQerrorMessage(m_localData->connection); + } + + ResultSet Connection::query (const std::string& parQuery) { + //TODO: sanitize input string + auto res = PQexec(m_localData->connection, parQuery.c_str()); + if (not res) + throw DatabaseException("Error running query", "Error allocating result object", __FILE__, __LINE__); + const int ress = PQresultStatus(res); + if (ress != PGRES_TUPLES_OK && ress != PGRES_COMMAND_OK) { + PQclear(res); + throw DatabaseException("Error running query", error_message(), __FILE__, __LINE__); + } + + ResultInfo info; + info.result = res; + info.column_mappings = nullptr; + return ResultSet(info); + } + + void Connection::query_void (const std::string& parQuery) { + //TODO: sanitize input string + auto deleter = [](PGresult* r) { PQclear(r); }; + std::unique_ptr res(PQexec(m_localData->connection, parQuery.c_str()), deleter); + if (not res) + throw DatabaseException("Error running query", "Error allocating result object", __FILE__, __LINE__); + const int ress = PQresultStatus(res.get()); + if (ress != PGRES_TUPLES_OK && ress != PGRES_COMMAND_OK) { + throw DatabaseException("Error running query", error_message(), __FILE__, __LINE__); + } + } +} //namespace pq diff --git a/src/pq/connection.hpp b/src/pq/connection.hpp new file mode 100644 index 0000000..7f630e1 --- /dev/null +++ b/src/pq/connection.hpp @@ -0,0 +1,51 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id68BF7AAC5BD54AF2BBD6EC1318B6687E +#define id68BF7AAC5BD54AF2BBD6EC1318B6687E + +#include "pq/resultset.hpp" +#include +#include +#include + +namespace pq { + class Connection { + public: + Connection ( std::string&& parUsername, std::string&& parPasswd, std::string&& parDatabase, std::string&& parAddress, uint16_t parPort ); + ~Connection ( void ) noexcept; + + std::string error_message ( void ) const; + bool is_connected ( void ) const noexcept; + void connect ( void ); + void disconnect ( void ); + void query_void ( const std::string& parQuery ); + ResultSet query ( const std::string& parQuery ); + + private: + struct LocalData; + + const std::string m_username; + const std::string m_passwd; + const std::string m_database; + const std::string m_address; + const uint16_t m_port; + std::unique_ptr m_localData; + }; +} //namespace pq + +#endif diff --git a/src/pq/databaseexception.cpp b/src/pq/databaseexception.cpp new file mode 100644 index 0000000..c7f5a9c --- /dev/null +++ b/src/pq/databaseexception.cpp @@ -0,0 +1,39 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include +#include "pq/databaseexception.hpp" + +namespace pq { + namespace { + std::string& compose_err_msg ( std::string& parMsg, const std::string& parErr, const std::string& parFile, std::size_t parLine) { + parMsg += ": "; + parMsg += parErr; + parMsg += '\n'; + parMsg += parFile; + parMsg += ':'; + parMsg += boost::lexical_cast(parLine); + return parMsg; + } + } //unnamed namespace + + DatabaseException::DatabaseException (std::string&& parMsg, std::string&& parErr, const std::string& parFile, std::size_t parLine) : + std::runtime_error(compose_err_msg(parMsg, parErr, parFile, parLine)), + m_error(std::move(parErr)) + { + } +} //namespace pq diff --git a/src/pq/resultinfo.hpp b/src/pq/resultinfo.hpp new file mode 100644 index 0000000..17ab374 --- /dev/null +++ b/src/pq/resultinfo.hpp @@ -0,0 +1,32 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idB756FAED6EE34FB18E302528CEA7C1D7 +#define idB756FAED6EE34FB18E302528CEA7C1D7 + +#include +#include +#include + +namespace pq { + struct ResultInfo { + PGresult* result; + const std::map* column_mappings; + }; +} //namespace pq + +#endif diff --git a/src/pq/resultset.cpp b/src/pq/resultset.cpp new file mode 100644 index 0000000..7f15710 --- /dev/null +++ b/src/pq/resultset.cpp @@ -0,0 +1,176 @@ +/* Copyright 2015, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "pq/resultset.hpp" +#include "pq/resultinfo.hpp" +#include "pq/databaseexception.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace pq { + struct Row::LocalData { + LocalData ( const PGresult* parResult, std::size_t parRow, const std::map* parCols, std::size_t parColCount ) : + result(parResult), + columns(parCols), + row_index(parRow), + col_count(parColCount) + { + assert((result and columns) or (not (result or columns))); + } + + const PGresult* const result; + const std::map* const columns; + const std::size_t row_index; + const std::size_t col_count; + }; + + struct ResultSet::LocalData { + explicit LocalData ( PGresult* parResult ) : + result(parResult) + { + } + + PGresult* result; + std::map columns; + }; + + Row::Row (const ResultInfo& parResult, std::size_t parRow) : + m_localData(new LocalData(parResult.result, parRow, parResult.column_mappings, (parResult.column_mappings ? parResult.column_mappings->size() : 0))) + { + assert(not m_localData->result or static_cast(std::max(0, PQnfields(m_localData->result))) == m_localData->col_count); + } + + Row::Row (Row&& parOther) : + m_localData(std::move(parOther.m_localData)) + { + } + + //Row::Row (const Row& parOther) : + // m_localData(new LocalData(parOther.m_localData->result, parOther.m_localData->row_index, parOther.m_localData->columns, parOther.m_localData->col_count)) + //{ + //} + + Row::~Row() noexcept { + } + + std::string Row::operator[] (const std::string& parIndex) const { + assert(m_localData->columns); + const auto num_index_it = m_localData->columns->find(parIndex); + if (num_index_it == m_localData->columns->end()) + throw DatabaseException("Error retrieving column", "Requested column \"" + parIndex + "\" doesn't exist", __FILE__, __LINE__); + + const int num_index = num_index_it->second; + return PQgetvalue(m_localData->result, static_cast(m_localData->row_index), num_index); + } + + std::size_t Row::size() const { + return m_localData->col_count; + } + + bool Row::empty() const { + return size() == 0; + } + + std::string Row::operator[] (std::size_t parIndex) const { + if (parIndex >= m_localData->col_count) { + std::ostringstream oss; + oss << "Requested column at index " << parIndex << " is out of range (column count is " << m_localData->col_count << ')'; + throw DatabaseException("Error retrieving column", oss.str(), __FILE__, __LINE__); + } + + return PQgetvalue(m_localData->result, static_cast(m_localData->row_index), static_cast(parIndex)); + } + + Row::const_iterator Row::begin() const { + return boost::make_transform_iterator( + boost::counting_iterator(0), + boost::bind(static_cast(&Row::operator[]), this, _1) + ); + } + + Row::const_iterator Row::end() const { + return boost::make_transform_iterator( + boost::counting_iterator(m_localData->col_count), + boost::bind(static_cast(&Row::operator[]), this, _1) + ); + } + + + ResultSet::ResultSet (ResultInfo& parResult) : + m_localData(new LocalData(parResult.result)) + { + const auto result = m_localData->result; + const int column_count = PQnfields(result); + std::string col_name; + for (int z = 0; z < column_count; ++z) { + m_localData->columns[PQfname(result, z)] = z; + } + } + + ResultSet::ResultSet (ResultSet&& parOther) : + m_localData(std::move(parOther.m_localData)) + { + } + + ResultSet::~ResultSet() noexcept { + if (m_localData->result) { + PQclear(m_localData->result); + m_localData->result = nullptr; + } + } + + Row ResultSet::operator[] (std::size_t parIndex) const { + ResultInfo info; + info.result = m_localData->result; + info.column_mappings = &m_localData->columns; + + if (parIndex >= size()) + throw DatabaseException("Out of range", "Column index " + boost::lexical_cast(parIndex) + " is out of range (column count is " + boost::lexical_cast(size()) + ')', __FILE__, __LINE__); + + return Row(info, parIndex); + } + + std::size_t ResultSet::size() const { + assert(m_localData->result); + const auto retval = PQntuples(m_localData->result); + assert(retval >= 0); + return static_cast(std::max(0, retval)); + } + + bool ResultSet::empty() const { + return size() == 0; + } + + ResultSet::const_iterator ResultSet::begin() const { + return boost::make_transform_iterator( + boost::counting_iterator(0), + boost::bind(static_cast(&ResultSet::operator[]), this, _1) + ); + } + + ResultSet::const_iterator ResultSet::end() const { + return boost::make_transform_iterator( + boost::counting_iterator(size()), + boost::bind(static_cast(&ResultSet::operator[]), this, _1) + ); + } +} //namespace pq diff --git a/src/tiger.cpp b/src/tiger.cpp index f822f81..963eb26 100644 --- a/src/tiger.cpp +++ b/src/tiger.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #if defined(__SSE2__) extern "C" void tiger_sse2_chunk ( const char* parStr1, const char* parStr2, uint64_t parLength, uint64_t parRes1[3], uint64_t parRes2[3] ); @@ -102,7 +103,13 @@ namespace din { std::string tiger_to_string (const TigerHash& parHash) { std::ostringstream oss; - oss << std::hex << swap_long(parHash.part_a) << swap_long(parHash.part_b) << swap_long(parHash.part_c); + oss << std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t)) + << swap_long(parHash.part_a) + << std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t)) + << swap_long(parHash.part_b) + << std::hex << std::setfill('0') << std::setw(2 * sizeof(uint64_t)) + << swap_long(parHash.part_c) + ; return oss.str(); }