King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-02-17 11:45:50 +00:00
Code Issues Releases Wiki Activity

Re-enable scan in the postgre plugin.

Browse source 
This also adds a public dependency to machinery in the header-only
backend project. This is because the backend does need functionality
from machinery, such as FileRecordData and TigerHash.
This commit is contained in:
King_DuckZ 2016-05-31 20:03:31 +02:00
parent 8f51f82abc
commit 3608d7b76c
9 changed files with 48 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CMakeLists.txt
View file

@ -132,7 +132,7 @@ add_subdirectory(src/backends)
#Actions
add_subdirectory(src/main)
#add_subdirectory(src/scan)
add_subdirectory(src/scan)
add_subdirectory(src/delete)
#add_subdirectory(src/query)
#add_subdirectory(src/locate)

4
include/backends/db_backend.hpp
View file

@ -19,6 +19,7 @@
#define id7506CA9825454B80856154ACFE8A9DE2
#include "backends/backend_loader.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include <cstdint>
#include <string>
#include <vector>
@ -48,6 +49,9 @@ namespace dindb {
virtual void delete_all_tags ( const std::vector<std::string>& parRegexes, GroupIDType parSet ) = 0;
virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) = 0;
virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature ) = 0;
virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) = 0;
};
} //namespace dindb

5
src/backends/CMakeLists.txt
View file

@ -13,8 +13,9 @@ target_include_directories(${PROJECT_NAME} SYSTEM
target_link_libraries(${PROJECT_NAME}
PRIVATE ${bare_name}-if
PUBLIC ${YAMLCPP_LIBRARY}
PUBLIC dl
INTERFACE ${YAMLCPP_LIBRARY}
INTERFACE dl
INTERFACE ${bare_name}-machinery
)
#install(TARGETS ${PROJECT_NAME}

2
src/backends/postgresql/CMakeLists.txt
View file

@ -4,7 +4,7 @@ add_library(${PROJECT_NAME} SHARED
tag.cpp
delete.cpp
#locate.cpp
#scan.cpp
scan.cpp
#dbsource.cpp
backend_postgresql.cpp
)

9
src/backends/postgresql/backend_postgresql.cpp
View file

@ -19,6 +19,7 @@
#include "backends/exposed_functions.hpp"
#include "tag.hpp"
#include "delete.hpp"
#include "scan.hpp"
#include "pq/connection.hpp"
#include <ciso646>
#include <utility>
@ -104,6 +105,14 @@ namespace dindb {
void BackendPostgreSql::delete_group (const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
dindb::delete_group_from_db(*m_conn, parIDs, parConf);
}
void BackendPostgreSql::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
dindb::write_to_db(*m_conn, parData, parSetData, parSignature);
}
bool BackendPostgreSql::search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
return dindb::read_from_db(parItem, parSet, *m_conn, parHash);
}
} //namespace dindb
extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {

3
src/backends/postgresql/backend_postgresql.hpp
View file

@ -43,6 +43,9 @@ namespace dindb {
virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) override;
virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash );
private:
std::unique_ptr<pq::Connection> m_conn;
};

28
src/backends/postgresql/scan.cpp
View file

@ -15,8 +15,7 @@
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "db/scan.hpp"
#include "db/settings.hpp"
#include "scan.hpp"
#include "pq/connection.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include <string>
@ -27,20 +26,20 @@
#include <memory>
#include <boost/utility/string_ref.hpp>
#include <chrono>
#include <cassert>
namespace dindb {
namespace {
} //unnamed namespace
bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const Settings& parDB, const mchlib::TigerHash& parHash) {
bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash) {
using boost::lexical_cast;
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
conn.connect();
assert(parDB.is_connected());
uint32_t group_id;
{
auto resultset = conn.query(
auto resultset = parDB.query(
"SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash=$1 LIMIT 1;",
tiger_to_string(parHash, true)
);
@ -64,7 +63,7 @@ namespace dindb {
}
{
auto resultset = conn.query(
auto resultset = parDB.query(
"SELECT \"desc\",\"type\",\"disk_number\" FROM sets WHERE \"id\"=$1;",
group_id
);
@ -82,21 +81,20 @@ namespace dindb {
return true;
}
void write_to_db (const Settings& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
void write_to_db (pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
using std::chrono::system_clock;
using boost::lexical_cast;
assert(parDB.is_connected());
if (parData.empty()) {
return;
}
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
conn.connect();
conn.query("BEGIN;");
parDB.query("BEGIN;");
uint32_t new_group_id;
{
auto id_res = conn.query("INSERT INTO \"sets\" "
auto id_res = parDB.query("INSERT INTO \"sets\" "
"(\"desc\",\"type\",\"app_name\""
",\"content_type\") "
"VALUES ($1, $2, $3, $4) RETURNING \"id\";",
@ -126,7 +124,7 @@ namespace dindb {
const auto& itm = parData[z];
assert(itm.path().data());
conn.query(query,
parDB.query(query,
(itm.path().empty() ? empty_path_string : itm.path()),
tiger_to_string(itm.hash),
itm.level,
@ -142,6 +140,6 @@ namespace dindb {
itm.mime_charset()
);
}
conn.query("COMMIT;");
parDB.query("COMMIT;");
}
} //namespace dindb

8
src/backends/postgresql/scan.hpp
View file

@ -29,11 +29,15 @@ namespace mchlib {
struct TigerHash;
} //namespace mchlib
namespace pq {
class Connection;
} //namespace pq
namespace dindb {
struct Settings;;
void write_to_db ( const Settings& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const Settings& parDB, const mchlib::TigerHash& parHash );
void write_to_db ( pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash );
} //namespace dindb
#endif

14
src/scan/main.cpp
View file

@ -25,7 +25,6 @@
#include "dindexer-common/common_info.hpp"
#include "dindexer-common/settings.hpp"
#include "commandline.hpp"
#include "db/scan.hpp"
#include "dindexer-machinery/scantask/dirtree.hpp"
#include "dindexer-machinery/scantask/mediatype.hpp"
#include "dindexer-machinery/scantask/hashing.hpp"
@ -41,7 +40,7 @@
#endif
namespace {
bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, const dindb::Settings& parDBSettings, bool parForce=false );
bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce=false );
#if defined(WITH_PROGRESS_FEEDBACK)
void print_progress ( const boost::string_ref parPath, uint64_t parFileBytes, uint64_t parTotalBytes, uint32_t parFileNum, std::size_t& parClearCount );
#endif
@ -81,6 +80,9 @@ int main (int parArgc, char* parArgv[]) {
return 1;
}
}
//TODO: throw if plugin loading failed
assert(settings.backend_plugin.name() == settings.backend_name);
assert(settings.backend_plugin.is_loaded());
bool ignore_read_errors = (vm.count("ignore-errors") > 0);
const std::string search_path(vm["search-path"].as<std::string>());
@ -104,7 +106,7 @@ int main (int parArgc, char* parArgv[]) {
);
#endif
const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.db);
const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.backend_plugin.backend());
#if defined(WITH_PROGRESS_FEEDBACK)
std::cout << '\n';
#endif
@ -115,7 +117,7 @@ int main (int parArgc, char* parArgv[]) {
}
namespace {
bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, const dindb::Settings& parDBSettings, bool parForce) {
bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce) {
using mchlib::FileRecordData;
using mchlib::SetRecordDataFull;
using mchlib::SetRecordData;
@ -124,7 +126,7 @@ namespace {
const auto& first_hash = parData.front().hash;
FileRecordData itm;
SetRecordDataFull set;
const bool already_in_db = dindb::read_from_db(itm, set, parDBSettings, first_hash);
const bool already_in_db = parDB.search_file_by_hash(itm, set, first_hash);
if (already_in_db) {
return false;
}
@ -134,7 +136,7 @@ namespace {
const auto app_signature = dinlib::dindexer_signature();
const auto lib_signature = mchlib::lib_signature();
const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
dindb::write_to_db(parDBSettings, parData, set_data, signature);
parDB.write_files(parData, set_data, signature);
return true;
}