Re-enable scan in the postgre plugin.

This also adds a public dependency to machinery in the header-only
backend project. This is because the backend does need functionality
from machinery, such as FileRecordData and TigerHash.

CMakeLists.txt

@@ -132,7 +132,7 @@ add_subdirectory(src/backends)
 
 #Actions
 add_subdirectory(src/main)
-#add_subdirectory(src/scan)
+add_subdirectory(src/scan)
 add_subdirectory(src/delete)
 #add_subdirectory(src/query)
 #add_subdirectory(src/locate)

include/backends/db_backend.hpp

@@ -19,6 +19,7 @@
 #define id7506CA9825454B80856154ACFE8A9DE2
 
 #include "backends/backend_loader.hpp"
+#include "dindexer-machinery/recorddata.hpp"
 #include <cstdint>
 #include <string>
 #include <vector>
@@ -48,6 +49,9 @@ namespace dindb {
 		virtual void delete_all_tags ( const std::vector<std::string>& parRegexes, GroupIDType parSet ) = 0;
 
 		virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) = 0;
+
+		virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature ) = 0;
+		virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) = 0;
 	};
 } //namespace dindb
 

src/backends/CMakeLists.txt

@@ -13,8 +13,9 @@ target_include_directories(${PROJECT_NAME} SYSTEM
 
 target_link_libraries(${PROJECT_NAME}
 	PRIVATE ${bare_name}-if
-	PUBLIC ${YAMLCPP_LIBRARY}
+	INTERFACE ${YAMLCPP_LIBRARY}
-	PUBLIC dl
+	INTERFACE dl
+	INTERFACE ${bare_name}-machinery
 )
 
 #install(TARGETS ${PROJECT_NAME}

src/backends/postgresql/CMakeLists.txt

@@ -4,7 +4,7 @@ add_library(${PROJECT_NAME} SHARED
 	tag.cpp
 	delete.cpp
 	#locate.cpp
-	#scan.cpp
+	scan.cpp
 	#dbsource.cpp
 	backend_postgresql.cpp
 )

src/backends/postgresql/backend_postgresql.cpp

@@ -19,6 +19,7 @@
 #include "backends/exposed_functions.hpp"
 #include "tag.hpp"
 #include "delete.hpp"
+#include "scan.hpp"
 #include "pq/connection.hpp"
 #include <ciso646>
 #include <utility>
@@ -104,6 +105,14 @@ namespace dindb {
 	void BackendPostgreSql::delete_group (const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
 		dindb::delete_group_from_db(*m_conn, parIDs, parConf);
 	}
+
+	void BackendPostgreSql::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
+		dindb::write_to_db(*m_conn, parData, parSetData, parSignature);
+	}
+
+	bool BackendPostgreSql::search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
+		return dindb::read_from_db(parItem, parSet, *m_conn, parHash);
+	}
 } //namespace dindb
 
 extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {

src/backends/postgresql/backend_postgresql.hpp

@@ -43,6 +43,9 @@ namespace dindb {
 
 		virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) override;
 
+		virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
+		virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash );
+
 	private:
 		std::unique_ptr<pq::Connection> m_conn;
 	};

src/backends/postgresql/scan.cpp

@@ -15,8 +15,7 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "db/scan.hpp"
+#include "scan.hpp"
-#include "db/settings.hpp"
 #include "pq/connection.hpp"
 #include "dindexer-machinery/recorddata.hpp"
 #include <string>
@@ -27,20 +26,20 @@
 #include <memory>
 #include <boost/utility/string_ref.hpp>
 #include <chrono>
+#include <cassert>
 
 namespace dindb {
 	namespace {
 	} //unnamed namespace
 
-	bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const Settings& parDB, const mchlib::TigerHash& parHash) {
+	bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash) {
 		using boost::lexical_cast;
 
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
+		assert(parDB.is_connected());
-		conn.connect();
 
 		uint32_t group_id;
 		{
-			auto resultset = conn.query(
+			auto resultset = parDB.query(
 				"SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash=$1 LIMIT 1;",
 				tiger_to_string(parHash, true)
 			);
@@ -64,7 +63,7 @@ namespace dindb {
 		}
 
 		{
-			auto resultset = conn.query(
+			auto resultset = parDB.query(
 				"SELECT \"desc\",\"type\",\"disk_number\" FROM sets WHERE \"id\"=$1;",
 				group_id
 			);
@@ -82,21 +81,20 @@ namespace dindb {
 		return true;
 	}
 
-	void write_to_db (const Settings& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
+	void write_to_db (pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
 		using std::chrono::system_clock;
 		using boost::lexical_cast;
 
+		assert(parDB.is_connected());
+
 		if (parData.empty()) {
 			return;
 		}
 
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
+		parDB.query("BEGIN;");
-		conn.connect();
-
-		conn.query("BEGIN;");
 		uint32_t new_group_id;
 		{
-			auto id_res = conn.query("INSERT INTO \"sets\" "
+			auto id_res = parDB.query("INSERT INTO \"sets\" "
 				"(\"desc\",\"type\",\"app_name\""
 				",\"content_type\") "
 				"VALUES ($1, $2, $3, $4) RETURNING \"id\";",
@@ -126,7 +124,7 @@ namespace dindb {
 
 			const auto& itm = parData[z];
 			assert(itm.path().data());
-			conn.query(query,
+			parDB.query(query,
 				(itm.path().empty() ? empty_path_string : itm.path()),
 				tiger_to_string(itm.hash),
 				itm.level,
@@ -142,6 +140,6 @@ namespace dindb {
 				itm.mime_charset()
 			);
 		}
-		conn.query("COMMIT;");
+		parDB.query("COMMIT;");
 	}
 } //namespace dindb

src/backends/postgresql/scan.hpp

@@ -29,11 +29,15 @@ namespace mchlib {
 	struct TigerHash;
 } //namespace mchlib
 
+namespace pq {
+	class Connection;
+} //namespace pq
+
 namespace dindb {
 	struct Settings;;
 
-	void write_to_db ( const Settings& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
+	void write_to_db ( pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
-	bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const Settings& parDB, const mchlib::TigerHash& parHash );
+	bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash );
 } //namespace dindb
 
 #endif

src/scan/main.cpp

@@ -25,7 +25,6 @@
 #include "dindexer-common/common_info.hpp"
 #include "dindexer-common/settings.hpp"
 #include "commandline.hpp"
-#include "db/scan.hpp"
 #include "dindexer-machinery/scantask/dirtree.hpp"
 #include "dindexer-machinery/scantask/mediatype.hpp"
 #include "dindexer-machinery/scantask/hashing.hpp"
@@ -41,7 +40,7 @@
 #endif
 
 namespace {
-	bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, const dindb::Settings& parDBSettings, bool parForce=false );
+	bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce=false );
 #if defined(WITH_PROGRESS_FEEDBACK)
 	void print_progress ( const boost::string_ref parPath, uint64_t parFileBytes, uint64_t parTotalBytes, uint32_t parFileNum, std::size_t& parClearCount );
 #endif
@@ -81,6 +80,9 @@ int main (int parArgc, char* parArgv[]) {
 			return 1;
 		}
 	}
+	//TODO: throw if plugin loading failed
+	assert(settings.backend_plugin.name() == settings.backend_name);
+	assert(settings.backend_plugin.is_loaded());
 
 	bool ignore_read_errors = (vm.count("ignore-errors") > 0);
 	const std::string search_path(vm["search-path"].as<std::string>());
@@ -104,7 +106,7 @@ int main (int parArgc, char* parArgv[]) {
 	);
 #endif
 
-	const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.db);
+	const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.backend_plugin.backend());
 #if defined(WITH_PROGRESS_FEEDBACK)
 		std::cout << '\n';
 #endif
@@ -115,7 +117,7 @@ int main (int parArgc, char* parArgv[]) {
 }
 
 namespace {
-	bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, const dindb::Settings& parDBSettings, bool parForce) {
+	bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce) {
 		using mchlib::FileRecordData;
 		using mchlib::SetRecordDataFull;
 		using mchlib::SetRecordData;
@@ -124,7 +126,7 @@ namespace {
 			const auto& first_hash = parData.front().hash;
 			FileRecordData itm;
 			SetRecordDataFull set;
-			const bool already_in_db = dindb::read_from_db(itm, set, parDBSettings, first_hash);
+			const bool already_in_db = parDB.search_file_by_hash(itm, set, first_hash);
 			if (already_in_db) {
 				return false;
 			}
@@ -134,7 +136,7 @@ namespace {
 		const auto app_signature = dinlib::dindexer_signature();
 		const auto lib_signature = mchlib::lib_signature();
 		const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
-		dindb::write_to_db(parDBSettings, parData, set_data, signature);
+		parDB.write_files(parData, set_data, signature);
 		return true;
 	}