Merge

.gitmodules

@@ -7,3 +7,9 @@
 [submodule "lib/better-enums"]
 	path = lib/better-enums
 	url = https://github.com/aantron/better-enums
+[submodule "lib/duckhandy"]
+	path = lib/duckhandy
+	url = https://github.com/KingDuckZ/duckhandy.git
+[submodule "lib/incredis"]
+	path = lib/incredis
+	url = ../incredis.git

CMakeLists.txt

@@ -21,6 +21,7 @@ include(gccversion)
 include(CPack)
 include(CTest)
 include(timestamp)
+include(shared_git_project)
 
 option(DINDEXER_DEBUG_CFG_FILE "Enable to set the config file path to the build path" OFF)
 option(DINDEXER_WITH_MEDIA_AUTODETECT "Enable code that tries to autodetect the media type and sets --type automatically" ON)
@@ -57,7 +58,7 @@ if ("${DINDEXER_CONFIG_FILE}" STREQUAL "")
 endif()
 message(STATUS "Config file set to \"${DINDEXER_CONFIG_FILE}\"")
 
-find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options)
+find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options filesystem system)
 find_package(PostgreSQL 8.3 REQUIRED)
 find_package(YamlCpp 0.5.1 REQUIRED)
 import_libpqtypes_project("${PostgreSQL_INCLUDE_DIRS}" "-O3 ${march_flag}")
@@ -87,7 +88,6 @@ configure_file(
 
 target_include_directories(${PROJECT_NAME} SYSTEM
 	INTERFACE ${Boost_INCLUDE_DIRS}
-	INTERFACE ${PostgreSQL_INCLUDE_DIRS}
 )
 
 target_compile_features(${PROJECT_NAME}
@@ -128,29 +128,35 @@ add_subdirectory(src/common)
 add_subdirectory(src/machinery)
 add_subdirectory(lib/pbl)
 add_subdirectory(lib/glob2regex)
+add_subdirectory(src/backends)
+add_subdirectory(src/core)
+add_shared_git_project(lib/duckhandy)
+add_shared_git_project(lib/incredis EXCLUDE_FROM_ALL)
 
 #Actions
 add_subdirectory(src/main)
 add_subdirectory(src/scan)
 add_subdirectory(src/delete)
-add_subdirectory(src/query)
+#add_subdirectory(src/query)
 add_subdirectory(src/locate)
 add_subdirectory(src/navigate)
 add_subdirectory(src/tag)
 
 #Tests
 if (BUILD_TESTING)
-	add_subdirectory(test/gtest)
+	add_shared_git_project(test/gtest)
 	add_subdirectory(test/unit)
 	add_subdirectory(test/unit_cli)
 endif()
 
 target_link_libraries(${PROJECT_NAME}
-	INTERFACE ${PostgreSQL_LIBRARIES}
 	INTERFACE ${Boost_LIBRARIES}
 	INTERFACE ${bare_name}-pq
 	INTERFACE ${bare_name}-inc
 )
+target_link_libraries(${bare_name}-inc
+	INTERFACE duckhandy
+)
 
 target_compile_definitions(${PROJECT_NAME}
 	INTERFACE WITH_PROGRESS_FEEDBACK

README.md

@@ -168,8 +168,6 @@ and the following libraries:
 * **machinery** the bulk of dindexer, such as the indexing functionality
 * **pq** a c++ wrapper for libpq (PostgreSql)
 
-Additionally inside include/helpers/ you will find header-only implementations of very generic helper tools.
-
 Currently, actions are just the command line frontend to access the functionalities in the libraries. This way writing a new CLI or GUI should be relatively easy.
 This structure might change in the future.
 

cmake/Modules/shared_git_project.cmake

@@ -0,0 +1,110 @@
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# For more information, please refer to <http://unlicense.org/>
+
+cmake_minimum_required(VERSION 2.8.12.2 FATAL_ERROR)
+
+function (add_shared_git_project SUBMODULE_PATH)
+	if (IS_ABSOLUTE "${SUBMODULE_PATH}")
+		set(submod_path "${SUBMODULE_PATH}")
+	else()
+		set(submod_path "${CMAKE_CURRENT_SOURCE_DIR}/${SUBMODULE_PATH}")
+	endif()
+
+	if (NOT EXISTS "${submod_path}")
+		message(FATAL_ERROR "Path \"${submod_path}\" doesn't exist")
+	endif()
+	if (NOT IS_DIRECTORY "${submod_path}")
+		message(FATAL_ERROR "Path \"${submod_path}\" is not a valid directory")
+	endif()
+
+	if (NOT EXISTS "${submod_path}/.git")
+		message(FATAL_ERROR ".git not found in \"${submod_path}\". Not a git submodule?")
+	endif()
+	if (IS_DIRECTORY "${submod_path}/.git")
+		message(FATAL_ERROR "\"${submod_path}.git\" is a directory, not a file as expected. Not a git submodule?")
+	endif()
+
+	get_filename_component(proj_name_orig "${submod_path}" NAME)
+	if ("${proj_name_orig}" STREQUAL "")
+		message(FATAL_ERROR "Couldn't make up a name for given project in \"${submod_path}\"")
+	endif()
+
+	string(MAKE_C_IDENTIFIER "${proj_name_orig}" proj_name_c_id)
+	string(TOUPPER ${proj_name_c_id} proj_name)
+
+	get_property(shared_projects_list GLOBAL PROPERTY SHARED_PROJECTS_LIST)
+	list(FIND shared_projects_list ${proj_name} found_index)
+	if (${found_index} GREATER -1)
+		#nothing to do, the variable is already set so the project must have been
+		#included already
+		return()
+	endif()
+
+	#Obtain the path to the working tree
+	# see http://stackoverflow.com/questions/27379818/git-possible-to-use-same-submodule-working-copy-by-multiple-projects
+	# git rev-parse --git-dir --show-toplevel
+	execute_process(
+		COMMAND git rev-parse --show-toplevel
+		WORKING_DIRECTORY "${submod_path}"
+		OUTPUT_VARIABLE matched_gitdir
+		OUTPUT_STRIP_TRAILING_WHITESPACE
+	)
+
+	#Make sure we got an absolute path
+	if (IS_ABSOLUTE "${matched_gitdir}")
+		set(reported_submodule_dir "${matched_gitdir}")
+	else()
+		file(RELATIVE_PATH reported_submodule_dir "${CMAKE_CURRENT_SOURCE_DIR}" "${submod_path}/${matched_gitdir}")
+	endif()
+	unset(matched_gitdir)
+
+	#Check if submodule is a subdirectory of the current source dir
+	file(RELATIVE_PATH reported_submodule_rel_path "${CMAKE_CURRENT_SOURCE_DIR}" "${reported_submodule_dir}")
+	string(LENGTH "${reported_submodule_rel_path}" rel_path_len)
+	if (${rel_path_len} GREATER 2)
+		string(SUBSTRING "${reported_submodule_rel_path}" 0 3 first_bit)
+		if ("../" STREQUAL "${first_bit}")
+			set(is_out_of_dirtree ON)
+		else()
+			set(is_out_of_dirtree OFF)
+		endif()
+		unset(first_bit)
+	else()
+		set(is_out_of_dirtree OFF)
+	endif()
+	unset(rel_path_len)
+
+	#Globally mark current submodule as handled
+	set_property(GLOBAL APPEND PROPERTY SHARED_PROJECTS_LIST ${proj_name})
+
+	set(shared_project_binary "${CMAKE_CURRENT_BINARY_DIR}/shared_projects/${proj_name_orig}")
+	if (is_out_of_dirtree)
+		#message(FATAL_ERROR "Would call add_subdirectory(\"${reported_submodule_dir}\" \"${shared_project_binary}\")")
+		add_subdirectory("${reported_submodule_dir}" "${shared_project_binary}")
+	else()
+		#message(FATAL_ERROR "Would call add_subdirectory(\"${reported_submodule_rel_path}\")")
+		add_subdirectory("${reported_submodule_rel_path}")
+	endif()
+endfunction()

dindexer.sql.in

@@ -5,7 +5,7 @@
 -- Dumped from database version 9.5.2
 -- Dumped by pg_dump version 9.5.2
 
--- Started on 2016-05-18 00:47:01 CEST
+-- Started on 2016-06-03 20:30:32 CEST
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -160,6 +160,8 @@ CREATE TABLE sets (
     creation timestamp with time zone DEFAULT now() NOT NULL,
     app_name character varying NOT NULL,
     content_type character(1) DEFAULT 'G'::bpchar NOT NULL,
+    disk_label text NOT NULL,
+    fs_uuid text NOT NULL,
     CONSTRAINT chk_sets_type CHECK (((type = 'C'::bpchar) OR (type = 'D'::bpchar) OR (type = 'V'::bpchar) OR (type = 'B'::bpchar) OR (type = 'F'::bpchar) OR (type = 'H'::bpchar) OR (type = 'Z'::bpchar) OR (type = 'O'::bpchar)))
 );
 
@@ -309,7 +311,7 @@ GRANT ALL ON SCHEMA public TO postgres;
 GRANT ALL ON SCHEMA public TO PUBLIC;
 
 
--- Completed on 2016-05-18 00:47:01 CEST
+-- Completed on 2016-06-03 20:30:33 CEST
 
 --
 -- PostgreSQL database dump complete

dindexer.yml

@@ -1,8 +1,21 @@
 %YAML 1.2
 ---
-db_settings:
-    username: your_username
-    password: your_password
-    dbname: dindexer
-    port: 5432
-    address: 200.100.200.100
+backend_name: postgresql
+postgresql_settings:
+    connection:
+        username: your_username
+        password: your_password
+        dbname: dindexer
+        port: 5432
+        address: 200.100.200.100
+
+redis_settings:
+    connection:
+        address: 200.100.200.100
+        port: 6379
+
+    script_paths:
+    - /home/duckz/dev/build/dindexer/Debug/src/backends/redis/lua
+    - /home/duckz/dev/build/dindexer/Release/src/backends/redis/lua
+
+backend_paths: path to your build_dir/src/backend

docs/hiredis_libev_multithread.md

@@ -0,0 +1,166 @@
+[TOC]
+
+# Using hiredis+libev in a separate thread for events
+
+Used sources:
+
+1. [stackoverflow.com/questions/14621261][link_so_1]
+2. [stackoverflow.com/questions/8611126][link_so_2]
+3. [libev thread locking example][link_ev_doc]
+
+
+## Explanation
+
+### Link 1
+
+Possibly due to the original question asked, [Link 1][link_so_1] seems to show that you need to have at least two ev_loop objects created in order to run one in a separate thread. The code on that page:
+
+```cpp
+//This program is demo for using pthreads with libev.
+//Try using Timeout values as large as 1.0 and as small as 0.000001
+//and notice the difference in the output
+
+//(c) 2009 debuguo
+//(c) 2013 enthusiasticgeek for stack overflow
+//Free to distribute and improve the code. Leave credits intact
+
+#include <ev.h>
+#include <stdio.h> // for puts
+#include <stdlib.h>
+#include <pthread.h>
+
+pthread_mutex_t lock;
+double timeout = 0.00001;
+ev_timer timeout_watcher;
+int timeout_count = 0;
+
+ev_async async_watcher;
+int async_count = 0;
+
+struct ev_loop* loop2;
+
+void* loop2thread(void* args)
+{
+    printf("Inside loop 2"); // Here one could initiate another timeout watcher
+    ev_loop(loop2, 0);       // similar to the main loop - call it say timeout_cb1
+    return NULL;
+}
+
+static void async_cb (EV_P_ ev_async *w, int revents)
+{
+    //puts ("async ready");
+    pthread_mutex_lock(&lock);     //Don't forget locking
+    ++async_count;
+    printf("async = %d, timeout = %d \n", async_count, timeout_count);
+    pthread_mutex_unlock(&lock);   //Don't forget unlocking
+}
+
+static void timeout_cb (EV_P_ ev_timer *w, int revents) // Timer callback function
+{
+    //puts ("timeout");
+    if (ev_async_pending(&async_watcher)==false) { //the event has not yet been processed (or even noted) by the event loop? (i.e. Is it serviced? If yes then proceed to)
+        ev_async_send(loop2, &async_watcher); //Sends/signals/activates the given ev_async watcher, that is, feeds an EV_ASYNC event on the watcher into the event loop.
+    }
+
+    pthread_mutex_lock(&lock);     //Don't forget locking
+    ++timeout_count;
+    pthread_mutex_unlock(&lock);   //Don't forget unlocking
+    w->repeat = timeout;
+    ev_timer_again(loop, &timeout_watcher); //Start the timer again.
+}
+
+int main (int argc, char** argv)
+{
+    if (argc < 2) {
+        puts("Timeout value missing.\n./demo <timeout>");
+        return -1;
+    }
+    timeout = atof(argv[1]);
+
+    struct ev_loop *loop = EV_DEFAULT;  //or ev_default_loop (0);
+
+    //Initialize pthread
+    pthread_mutex_init(&lock, NULL);
+    pthread_t thread;
+
+    // This loop sits in the pthread
+    loop2 = ev_loop_new(0);
+
+    //This block is specifically used pre-empting thread (i.e. temporary interruption and suspension of a task, without asking for its cooperation, with the intention to resume that task later.)
+    //This takes into account thread safety
+    ev_async_init(&async_watcher, async_cb);
+    ev_async_start(loop2, &async_watcher);
+    pthread_create(&thread, NULL, loop2thread, NULL);
+
+    ev_timer_init (&timeout_watcher, timeout_cb, timeout, 0.); // Non repeating timer. The timer starts repeating in the timeout callback function
+    ev_timer_start (loop, &timeout_watcher);
+
+    // now wait for events to arrive
+    ev_loop(loop, 0);
+    //Wait on threads for execution
+    pthread_join(thread, NULL);
+
+    pthread_mutex_destroy(&lock);
+    return 0;
+}
+```
+
+with the comment *"Note for libev 4+ ev_loop should be ev_run."* is still slightly useful but it shouldn't be taken as a model.
+
+
+### Link 2
+
+The [second link][link_so_2] is what shows that one ev_loop is enough. This is the code provided by the original poster:
+
+```cpp
+    void RedisSubscriber::Start() {
+        m_redis = redisAsyncConnect(m_addr.c_str(),m_port);
+        m_redis->data = (void*)this;
+
+        m_loop = ev_loop_new(EVFLAG_NOINOTIFY);
+        redisLibevAttach(m_loop, m_redis);
+        redisAsyncSetConnectCallback(m_redis,connectCallback);
+        redisAsyncSetDisconnectCallback(m_redis,disconnectCallback);
+        redisAsyncCommand(m_redis, subscribeCallback, NULL, "SUBSCRIBE %s", m_channel.c_str());
+
+        m_thread = boost::thread(ev_loop,m_loop,0);
+    }
+
+    void RedisSubscriber::Stop() {
+        redisAsyncFree(m_redis);
+        m_thread.join();
+        m_redis = 0;
+    }
+
+    void RedisSubscriber::connectCallback(const redisAsyncContext *c) {
+
+    }
+
+    void RedisSubscriber::disconnectCallback(const redisAsyncContext *c, int status) {
+        RedisSubscriber* r = (RedisSubscriber*)(c->data);
+        ev_unloop(r->m_loop,EVUNLOOP_ALL);
+    }
+
+    void RedisSubscriber::subscribeCallback(redisAsyncContext *c, void *r, void *privdata) {
+
+    }
+```
+
+There are no accepted answers, but the answer from *themoondothshine* provides very useful info. Here is what it says:
+
+Assuming that you mean ev_run for your boost::thread, here's what you can do:
+
+1. Setup an `ev_async`
+2. In the callback of `ev_async` call `ev_break`.
+3. Call `ev_async_send` from `RedisSubscriber::Stop()`. `ev_async` watchers are thread-safe -- it uses memory barriers for synchronising between threads.
+
+This will cause the event loop to stop, and `m_thread.join()` will return.
+
+
+### Link 3
+
+The [THREAD LOCKING EXAMPLE][link_ev_doc] shows how to lock in order to protect the ev_loop object in use.
+
+[link_so_1]: http://stackoverflow.com/questions/14621261/using-libev-with-multiple-threads#14779930
+[link_so_2]: http://stackoverflow.com/questions/8611126/hiredis-libev-and-boostthreads
+[link_ev_doc]: http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod#THREAD_LOCKING_EXAMPLE

docs/posts/04_whats_cooking.md

@@ -0,0 +1,11 @@
+### What's cooking? ###
+
+It's been roughly a month since I released version 0.1.5b and most of the activity has been on the [dev branch](https://bitbucket.org/King_DuckZ/dindexer/branch/dev) since then. It's still a bit too early to make a new release, but if you want to get a peek at the new features this is the right moment as commit e2275ce should give you a working version of #dindexer.
+
+Most notably, you will have a `tag` command, that will allow you to tag files. You should also be able to list files by tag, using the `dindexer locate --tags <my_tags>` command.
+
+The less evident change (although that's what took up most of my time) is the separation of the PostgreSQL code from the CL client. You have been asking for Sqlite support, I heard you and I'm getting everything ready so that an Sqlite plugin can be implemented! Currently, PostgreSQL is still the only backend you can choose, so please be patient... or even better, step in and contribute to dindexer! :)
+
+[bitbucket.org/King_DuckZ/dindexer](https://bitbucket.org/King_DuckZ/dindexer)
+
+\#dindexer #linux #opensource #cpp #sqlite

flat_git.yml

@@ -0,0 +1,3 @@
+inplace_submodules:
+    - pbl
+    - better-enums

include/backends/backend_loader.hpp

@@ -0,0 +1,62 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id756A258A98B24B0DB2529BCEEC5137E2
+#define id756A258A98B24B0DB2529BCEEC5137E2
+
+#include <string>
+#include <memory>
+#include <boost/utility/string_ref.hpp>
+
+namespace YAML {
+	class Node;
+} //namespace YAML
+
+namespace dindb {
+	class Backend;
+
+	using BackendPtr = std::unique_ptr<dindb::Backend, void(*)(dindb::Backend*)>;
+
+	class BackendPlugin {
+	public:
+		BackendPlugin ( void );
+		BackendPlugin ( BackendPlugin&& ) = default;
+		BackendPlugin ( const std::string& parSOPath, const YAML::Node* parConfig );
+		~BackendPlugin ( void ) noexcept;
+
+		const boost::string_ref& name ( void ) const;
+		Backend& backend ( void );
+		const Backend& backend ( void ) const;
+		bool is_loaded ( void ) const;
+		int backend_interface_version ( void ) const;
+		int max_supported_interface_version ( void ) const;
+
+		BackendPlugin& operator= ( BackendPlugin&& ) = default;
+
+	private:
+		using SoHandle = std::unique_ptr<void, int(*)(void*)>;
+
+		SoHandle m_lib;
+		BackendPtr m_backend;
+		boost::string_ref m_name;
+		int m_iface_ver;
+	};
+
+	std::string backend_name ( const std::string& parSOPath );
+} //namespace dindb
+
+#endif

include/backends/backend_version.hpp

@@ -15,24 +15,11 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef id9B6B373E88404330ADEE51A4EC861787
-#define id9B6B373E88404330ADEE51A4EC861787
+#ifndef idAA27B58429DB41C2AF53204CC5010E94
+#define idAA27B58429DB41C2AF53204CC5010E94
 
-#if !defined(CMAKE_ON)
-#	define CMAKE_ON 1
-#	define CMAKE_on 1
-#	define CMAKE_On 1
-#	define CMAKE_oN 1
-#endif
-#if !defined(CMAKE_OFF)
-#	define CMAKE_OFF 0
-#	define CMAKE_OFf 0
-#	define CMAKE_OfF 0
-#	define CMAKE_Off 0
-#	define CMAKE_oFF 0
-#	define CMAKE_oFf 0
-#	define CMAKE_ofF 0
-#	define CMAKE_off 0
-#endif
+namespace dindb {
+	constexpr const int g_current_iface_version = 1;
+} //namespace dindb
 
 #endif

include/backends/db_backend.hpp

@@ -0,0 +1,91 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id7506CA9825454B80856154ACFE8A9DE2
+#define id7506CA9825454B80856154ACFE8A9DE2
+
+#include "backends/backend_loader.hpp"
+#include "duckhandy/MaxSizedArray.hpp"
+#include <cstdint>
+#include <string>
+#include <vector>
+#include <boost/utility/string_ref.hpp>
+#include <map>
+#include <functional>
+
+namespace mchlib {
+	struct TigerHash;
+	struct FileRecordData;
+	struct SetRecordDataFull;
+} //namespace mchlib
+
+namespace dindb {
+	using GroupIDType = uint32_t;
+	using FileIDType = uint64_t;
+	using IDDescMap = std::map<uint32_t, std::string>;
+	using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
+	using TagList = std::vector<boost::string_ref>;
+
+	constexpr const GroupIDType InvalidGroupID = 0;
+	constexpr const FileIDType InvalidFileID = 0;
+
+	struct LocatedItem {
+		std::string path;
+		FileIDType id;
+		GroupIDType group_id;
+	};
+
+	struct LocatedSet {
+		std::string desc;
+		GroupIDType id;
+		uint32_t files_count;
+		uint32_t dir_count;
+	};
+
+	class Backend {
+	public:
+		Backend ( void ) = default;
+		virtual ~Backend ( void ) noexcept = default;
+
+		virtual void connect ( void ) = 0;
+		virtual void disconnect ( void ) = 0;
+
+		virtual void tag_files ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) = 0;
+		virtual void tag_files ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) = 0;
+		virtual void delete_tags ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) = 0;
+		virtual void delete_tags ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) = 0;
+		virtual void delete_all_tags ( const std::vector<FileIDType>& parFiles, GroupIDType parSet ) = 0;
+		virtual void delete_all_tags ( const std::vector<std::string>& parRegexes, GroupIDType parSet ) = 0;
+
+		virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) = 0;
+
+		virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) = 0;
+		virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) = 0;
+
+		virtual std::vector<LocatedItem> locate_in_db ( const std::string& parSearch, const TagList& parTags ) = 0;
+		virtual std::vector<LocatedItem> locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) = 0;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive ) = 0;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive ) = 0;
+
+		virtual std::vector<GroupIDType> find_all_sets ( void ) = 0;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 4>> find_set_details ( const std::vector<GroupIDType>& parSets ) = 0;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 1>> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) = 0;
+		virtual std::vector<std::string> find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) = 0;
+	};
+} //namespace dindb
+
+#endif

include/backends/exposed_functions.hpp

@@ -15,23 +15,20 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef id19B690A53A9546D5BD95D89FFF388283
-#define id19B690A53A9546D5BD95D89FFF388283
+#ifndef idA9E47E37E2FA49EE84C2E93FB701C368
+#define idA9E47E37E2FA49EE84C2E93FB701C368
 
-#if defined(__cplusplus)
-#	include <cstddef>
-#else
-#	include <stddef.h>
-#endif
+namespace YAML {
+	class Node;
+} //namespace YAML
 
-#if defined(lengthof)
-#	undef lengthof
-#endif
-//http://stackoverflow.com/questions/4415524/common-array-length-macro-for-c#4415646
-#if defined(__cplusplus)
-#	define lengthof(x) ((sizeof(x)/sizeof(0[x])) / ((std::size_t)(!(sizeof(x) % sizeof(0[x])))))
-#else
-#	define lengthof(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x])))))
-#endif
+namespace dindb {
+	class Backend;
+} //namespace dindb
+
+extern "C" dindb::Backend* dindexer_create_backend ( const YAML::Node* parConfig );
+extern "C" void dindexer_destroy_backend ( dindb::Backend* parDele );
+extern "C" const char* dindexer_backend_name ( void );
+extern "C" int dindexer_backend_iface_version ( void );
 
 #endif

include/dindexer-common/settings.hpp

@@ -19,22 +19,16 @@
 #define idDC29E3C667BD4793BA0644AE7DC5BD3F
 
 #include <string>
-#include <cstdint>
+#include "backends/backend_loader.hpp"
+#include "backends/db_backend.hpp"
 
 namespace dinlib {
-	struct SettingsDB {
-		std::string address;
-		std::string username;
-		std::string password;
-		std::string dbname;
-		uint16_t port;
-	};
-
 	struct Settings {
-		SettingsDB db;
+		std::string backend_name;
+		dindb::BackendPlugin backend_plugin;
 	};
 
-	bool load_settings ( const std::string& parPath, Settings& parOut, bool parExpand=true );
+	void load_settings ( const std::string& parPath, Settings& parOut, bool parExpand=true );
 } //namespace dinlib
 
 #endif

include/dindexer-core/searchpaths.hpp

@@ -0,0 +1,66 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idDD90110B9C7E44C4AAF8B46A663B11DC
+#define idDD90110B9C7E44C4AAF8B46A663B11DC
+
+#include <vector>
+#include <initializer_list>
+#include <string>
+#include <boost/utility/string_ref.hpp>
+#include <functional>
+
+namespace dincore {
+	class SearchPaths {
+	public:
+		enum SearchType {
+			Directory,
+			File,
+			Any
+		};
+		using Predicate = std::function<bool(boost::string_ref, const std::string& parPath)>;
+
+		SearchPaths ( void ) = default;
+		explicit SearchPaths ( std::vector<std::string>&& parList );
+		SearchPaths ( std::initializer_list<std::string> parInit );
+		~SearchPaths ( void ) noexcept;
+
+		void add_path ( std::string&& parPath );
+		std::string first_hit ( boost::string_ref parFile, SearchType parType=Any ) const;
+		std::string first_hit ( Predicate parPredicate, SearchType parType=Any ) const;
+
+	private:
+		std::vector<std::string> m_paths;
+	};
+
+	class ShallowSearchPaths {
+	public:
+		ShallowSearchPaths ( void ) = default;
+		explicit ShallowSearchPaths ( std::vector<boost::string_ref>&& parList );
+		ShallowSearchPaths ( std::initializer_list<boost::string_ref> parInit );
+		~ShallowSearchPaths ( void ) noexcept;
+
+		void add_path ( boost::string_ref parPath );
+		std::string first_hit ( boost::string_ref parFile, SearchPaths::SearchType parType=SearchPaths::Any ) const;
+		std::string first_hit ( SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType=SearchPaths::Any ) const;
+
+	private:
+		std::vector<boost::string_ref> m_paths;
+	};
+} //namespace dincore
+
+#endif

include/dindexer-core/split_tags.hpp

@@ -21,10 +21,11 @@
 #include <vector>
 #include <string>
 #include <boost/utility/string_ref.hpp>
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 
-namespace dinlib {
+namespace dincore {
+	std::vector<boost::string_ref> split_and_trim ( const std::string& parList, char parSeparator ) a_pure;
 	std::vector<boost::string_ref> split_tags ( const std::string& parCommaSeparatedList ) a_pure;
-} //namespace dinlib
+} //namespace dincore
 
 #endif

include/dindexer-machinery/guess_content_type.hpp

@@ -20,7 +20,7 @@
 
 #include "dindexer-machinery/mediatypes.hpp"
 #include "dindexer-machinery/recorddata.hpp"
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 #include <vector>
 
 namespace mchlib {

include/dindexer-machinery/mediatypes.hpp

@@ -18,7 +18,7 @@
 #ifndef id700AFD0F33634ACC88079BB8853A9E13
 #define id700AFD0F33634ACC88079BB8853A9E13
 
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 #include "enum.h"
 #include <string>
 

include/dindexer-machinery/recorddata.hpp

@@ -34,7 +34,7 @@ namespace mchlib {
 		typedef boost::flyweight<std::string, boost::flyweights::no_locking, MimeStringTag> mime_string;
 
 		FileRecordData ( void ) = default;
-		FileRecordData ( const char* parPath, std::time_t parATime, std::time_t parMTime, uint64_t parLevel, bool parIsDir, bool parIsSymLink ) :
+		FileRecordData ( const char* parPath, std::time_t parATime, std::time_t parMTime, uint16_t parLevel, bool parIsDir, bool parIsSymLink ) :
 			hash {},
 			abs_path(parPath),
 			mime_full(),
@@ -87,9 +87,21 @@ namespace mchlib {
 		bool operator== ( const FileRecordData& parOther ) const;
 #endif
 
-		boost::string_ref path ( void ) const { return boost::string_ref(abs_path).substr(path_offset); }
-		boost::string_ref mime_type ( void ) const { return boost::string_ref(mime_full.get()).substr(mime_type_offset, mime_type_length); }
-		boost::string_ref mime_charset ( void ) const { return boost::string_ref(mime_full.get()).substr(mime_charset_offset, mime_charset_length); }
+		boost::string_ref path ( void ) const {
+			assert(abs_path.data());
+			return boost::string_ref(abs_path).substr(path_offset);
+		}
+
+		boost::string_ref mime_type ( void ) const {
+			assert(mime_full.get().data());
+			return boost::string_ref(mime_full.get()).substr(mime_type_offset, mime_type_length);
+		}
+
+		boost::string_ref mime_charset ( void ) const {
+			assert(mime_full.get().data());
+			return boost::string_ref(mime_full.get()).substr(mime_charset_offset, mime_charset_length);
+		}
+
 		void set_mime_parts ( boost::string_ref parType, boost::string_ref parCharset ) {
 			const auto& mime = mime_full.get();
 			{
@@ -130,17 +142,15 @@ namespace mchlib {
 		bool hash_valid;
 	};
 
-	struct SetRecordData {
-		boost::string_ref name;
+	struct SetRecordDataFull {
+		std::string name;
+		std::string disk_label;
+		std::string fs_uuid;
+		uint32_t disk_number;
 		char type;
 		char content_type;
 	};
 
-	struct SetRecordDataFull : public SetRecordData {
-		std::string name;
-		uint32_t disk_number;
-	};
-
 #if !defined(NDEBUG)
 	inline bool FileRecordData::operator== (const FileRecordData& parOther) const {
 		return (this->abs_path == parOther.abs_path);

include/dindexer-machinery/tiger.hpp

@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <string>
 #include <vector>
+#include <ciso646>
 
 namespace mchlib {
 	struct TigerHash {
@@ -35,6 +36,8 @@ namespace mchlib {
 			uint64_t data[3];
 			uint8_t byte_data[sizeof(uint64_t) * 3];
 		};
+
+		bool operator== (const TigerHash& parOther) const { return part_a == parOther.part_a and part_b == parOther.part_b and part_c == parOther.part_c; }
 	};
 
 	static_assert(sizeof(TigerHash) == 24, "Wrong struct size");
@@ -42,6 +45,7 @@ namespace mchlib {
 	void tiger_file ( const std::string& parPath, TigerHash& parHashFile, TigerHash& parHashDir, uint64_t& parSizeOut );
 	void tiger_init_hash ( TigerHash& parHash );
 	std::string tiger_to_string ( const TigerHash& parHash, bool parUpcase=false );
+	TigerHash string_to_tiger ( const std::string& parString );
 	void tiger_data ( const std::string& parData, TigerHash& parHash );
 	void tiger_data ( const std::vector<char>& parData, TigerHash& parHash );
 } //namespace mchlib

include/helpers/compatibility.h

@@ -1,62 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef id45CDD1DAEF4F42968E3C89F68FDDA9BC
-#define id45CDD1DAEF4F42968E3C89F68FDDA9BC
-
-#if defined(__GNUC__)
-#	if defined(__clang__)
-#		if !defined(__has_attribute)
-			//Fall back to version number comparing
-#		else
-#			if __has_attribute(flatten)
-#				define a_flatten __attribute__((flatten))
-#			else
-#				define a_flatten
-#			endif
-#			if __has_attribute(always_inline)
-#				define a_always_inline __attribute__((always_inline))
-#			else
-#				define a_always_inline
-#			endif
-#			if __has_attribute(pure)
-#				define a_pure __attribute__((pure))
-#			else
-#				define a_pure
-#			endif
-#			if __has_attribute(deprecated)
-#				define a_deprecated __attribute__((deprecated))
-#			else
-#				define a_deprecated
-#endif
-#		endif
-#	else
-		//Fix here if you get warnings about unsupported attributes on your compiler
-#		define a_flatten __attribute__((flatten))
-#		define a_always_inline __attribute__((always_inline))
-#		define a_pure __attribute__((pure))
-#		define a_deprecated __attribute__((deprecated))
-#	endif
-#else
-#	warning "Unsupported compiler, please fill this section or file a bug"
-#	define a_flatten
-#	define a_always_inline
-#	define a_pure
-#	define a_deprecated
-#endif
-
-#endif

include/helpers/infix_iterator.hpp

@@ -1,48 +0,0 @@
-// see http://stackoverflow.com/questions/3496982/printing-lists-with-commas-c/3497021#3497021
-// infix_iterator.h
-//
-// Lifted from Jerry Coffin's 's prefix_ostream_iterator
-#if !defined(INFIX_ITERATOR_H_)
-#define  INFIX_ITERATOR_H_
-#include <ostream>
-#include <iterator>
-template <class T,
-          class charT=char,
-          class traits=std::char_traits<charT> >
-class infix_ostream_iterator :
-    public std::iterator<std::output_iterator_tag,void,void,void,void>
-{
-    std::basic_ostream<charT,traits> *os;
-    charT const* delimiter;
-    bool first_elem;
-public:
-    typedef charT char_type;
-    typedef traits traits_type;
-    typedef std::basic_ostream<charT,traits> ostream_type;
-    infix_ostream_iterator(ostream_type& s)
-        : os(&s),delimiter(0), first_elem(true)
-    {}
-    infix_ostream_iterator(ostream_type& s, charT const *d)
-        : os(&s),delimiter(d), first_elem(true)
-    {}
-    infix_ostream_iterator<T,charT,traits>& operator=(T const &item)
-    {
-        // Here's the only real change from ostream_iterator:
-        // Normally, the '*os << item;' would come before the 'if'.
-        if (!first_elem && delimiter != 0)
-            *os << delimiter;
-        *os << item;
-        first_elem = false;
-        return *this;
-    }
-    infix_ostream_iterator<T,charT,traits> &operator*() {
-        return *this;
-    }
-    infix_ostream_iterator<T,charT,traits> &operator++() {
-        return *this;
-    }
-    infix_ostream_iterator<T,charT,traits> &operator++(int) {
-        return *this;
-    }
-};
-#endif

include/pq/implem/pq_type_helpers.hpp

@@ -18,7 +18,7 @@
 #ifndef idEC73C3E4D64D44ABA0DB7D41FA8A7EB7
 #define idEC73C3E4D64D44ABA0DB7D41FA8A7EB7
 
-#include "pq/implem/string_bt.hpp"
+#include "duckhandy/string_bt.hpp"
 #include <chrono>
 #include <type_traits>
 #include <boost/utility/string_ref.hpp>
@@ -30,6 +30,8 @@ struct pg_param;
 typedef pg_param PGparam;
 
 namespace pq {
+	namespace bt = dhandy::bt;
+
 	class Connection;
 
 	namespace implem {

include/pq/implem/sequence_bt.hpp

@@ -1,48 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef id4FAEF395B9ED47CB9D6B50B54C9A289A
-#define id4FAEF395B9ED47CB9D6B50B54C9A289A
-
-#include <cstddef>
-
-namespace pq {
-	namespace bt {
-		template <std::size_t... I>
-		struct index_seq {
-		};
-
-		namespace implem {
-			template <std::size_t MIN, std::size_t MAX, std::size_t... I>
-			struct range_builder;
-
-			template <std::size_t MIN, std::size_t... I>
-			struct range_builder<MIN, MIN, I...> {
-				typedef index_seq<I...> type;
-			};
-
-			template <std::size_t MIN, std::size_t N, std::size_t... I>
-			struct range_builder : public range_builder<MIN, N - 1, N - 1, I...> {
-			};
-		} //namespace implem
-
-		template <std::size_t MIN, std::size_t MAX>
-		using index_range = typename implem::range_builder<MIN, MAX>::type;
-	} //namespace bt
-} //namespace pq
-
-#endif

include/pq/implem/string_bt.hpp

@@ -1,112 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef id170B0E6C34D14EBA9B92A35977BDBFB3
-#define id170B0E6C34D14EBA9B92A35977BDBFB3
-
-#include "pq/implem/sequence_bt.hpp"
-#include <cstddef>
-#include <iostream>
-#include <stdexcept>
-
-namespace pq {
-	namespace bt {
-		template <std::size_t S>
-		class string;
-
-		template <std::size_t S>
-		std::ostream& operator<< ( std::ostream& parStream, const string<S>& parString );
-
-		template <std::size_t S>
-		class string {
-		public:
-			friend std::ostream& operator<< <>( std::ostream& parStream, const string<S>& parString );
-			constexpr string ( const char* parString );
-
-			constexpr std::size_t size ( void ) const { return S - 1; }
-			template <std::size_t S2>
-			constexpr string<S + S2 - 1> operator+ ( const string<S2>& parOther ) const;
-			constexpr char operator[] ( std::size_t parIndex ) const;
-
-			template <typename... Args>
-			constexpr string ( Args... );
-
-			constexpr const char (&data_arr() const)[S] { return m_data; }
-			constexpr const char* data() const { return m_data; }
-
-		private:
-			template <std::size_t... I>
-			constexpr string ( const index_seq<I...>&, const char* parString );
-
-			const char m_data[S];
-		};
-
-		namespace implem {
-			template <std::size_t S, std::size_t S2, std::size_t... I>
-			constexpr string<S + S2 - 1> concat ( const index_seq<I...>&, const string<S>& parLeft, const string<S2>& parRight ) {
-				return string<S + S2 - 1>(
-					(I < S - 1 ? parLeft[I] : (I < S + S2 - 2 ? parRight[I - (S - 1)] : '\0'))...
-				);
-			}
-
-		} //namespace implem
-
-		template <std::size_t S>
-		template <std::size_t... I>
-		constexpr string<S>::string (const index_seq<I...>&, const char* parString) :
-			m_data{parString[I]...}
-		{
-		}
-
-		template <std::size_t S>
-		inline constexpr string<S>::string (const char* parString) :
-			string(index_range<0, S>(), parString)
-		{
-		}
-
-		template <std::size_t S>
-		template <typename... Args>
-		inline constexpr string<S>::string (Args... parArgs) :
-			m_data{parArgs...}
-		{
-		}
-
-		template <std::size_t S>
-		template <std::size_t S2>
-		constexpr inline string<S + S2 - 1> string<S>::operator+ (const string<S2>& parOther) const {
-			return implem::concat(index_range<0, S + S2 - 1>(), string<S>(m_data), parOther);
-		}
-
-		template <std::size_t S>
-		inline std::ostream& operator<< (std::ostream& parStream, const string<S>& parString) {
-			parStream << parString.m_data;
-			return parStream;
-		}
-
-		template <std::size_t S>
-		constexpr char string<S>::operator[] (std::size_t parIndex) const {
-			return (parIndex < S ? m_data[parIndex] : throw std::out_of_range(""));
-		}
-
-		template <std::size_t S>
-		constexpr string<S> make_string (const char (&parData)[S]) {
-			return string<S>(parData);
-		}
-	} //namespace bt
-} //namespace pq
-
-#endif

lib/duckhandy

@@ -0,0 +1 @@
+Subproject commit 9587e0e7a19275c462e22b044e9511bac9597605

lib/incredis

@@ -0,0 +1 @@
+Subproject commit 94fa688c1784b49689c9a057566bf281016c1634

src/backends/CMakeLists.txt

@@ -0,0 +1,37 @@
+project(${bare_name}-backend CXX)
+
+set(BACKEND_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+set(DINDEXER_ENABLED_BACKENDS "postgresql" CACHE STRING "Comma separated list of enabled backends. See directories in backends source directory for a list of known backends.")
+
+function(ln_backend backend_name)
+	add_custom_command(TARGET "${backend_name}" POST_BUILD
+		COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:${backend_name}> ${BACKEND_BINARY_DIR}/$<TARGET_FILE_NAME:${backend_name}>
+		DEPENDS ${BACKEND_BINARY_DIR}/$<TARGET_FILE_NAME:${backend_name}>
+		COMMENT "Creating symlink to ${backend_name}"
+	)
+endfunction()
+
+add_library(${PROJECT_NAME} STATIC
+	backend_loader.cpp
+)
+
+target_include_directories(${PROJECT_NAME}
+	PUBLIC ${DINDEXER_PUB_INCLUDE_DIR}
+)
+target_include_directories(${PROJECT_NAME} SYSTEM
+	PUBLIC ${YAMLCPP_INCLUDE_DIR}
+)
+
+target_link_libraries(${PROJECT_NAME}
+	PRIVATE ${bare_name}-if
+	INTERFACE ${YAMLCPP_LIBRARY}
+	INTERFACE dl
+	INTERFACE ${bare_name}-machinery
+)
+
+string(REPLACE "," ";" backend_list "${DINDEXER_ENABLED_BACKENDS}")
+foreach(backend ${backend_list})
+	add_subdirectory(${backend})
+	add_dependencies(${PROJECT_NAME} ${bare_name}-backend-${backend})
+endforeach()
+unset(backend)

src/backends/backend_loader.cpp

@@ -0,0 +1,145 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//See:
+// http://stackoverflow.com/questions/496664/c-dynamic-shared-library-on-linux
+
+#include "backends/backend_loader.hpp"
+#include "backends/exposed_functions.hpp"
+#include "backends/backend_version.hpp"
+#include <dlfcn.h>
+#include <cassert>
+#include <functional>
+#include <sstream>
+
+namespace dindb {
+	namespace {
+		BackendPtr load_backend (void* parSOHandle, const YAML::Node* parConfig) {
+			typedef decltype(&dindexer_create_backend) CreateBackendFun;
+			typedef decltype(&dindexer_destroy_backend) DeleteBackendFun;
+
+			assert(parSOHandle);
+			assert(parConfig);
+
+			const char* const fun_name_create = "dindexer_create_backend";
+			const char* const fun_name_destroy = "dindexer_destroy_backend";
+
+			auto create = reinterpret_cast<CreateBackendFun>(dlsym(parSOHandle, fun_name_create));
+			auto destroy = reinterpret_cast<DeleteBackendFun>(dlsym(parSOHandle, fun_name_destroy));
+
+			if (not create) {
+				std::ostringstream oss;
+				oss << "Unable to find function " << fun_name_create;
+				throw std::runtime_error(oss.str());
+			}
+			if (not destroy) {
+				std::ostringstream oss;
+				oss << "Unable to find function " << fun_name_destroy;
+				throw std::runtime_error(oss.str());
+			}
+			return BackendPtr(create(parConfig), destroy);
+		}
+
+		const char* backend_name (void* parSOHandle) {
+			typedef decltype(&dindexer_backend_name) GetNameFun;
+
+			assert(parSOHandle);
+
+			auto get_name = reinterpret_cast<GetNameFun>(dlsym(parSOHandle, "dindexer_backend_name"));
+			return get_name();
+		}
+
+		void nop_destroy (Backend*) {
+		}
+
+		int backend_iface_version (void* parSOHandle) {
+			typedef decltype(&dindexer_backend_iface_version) GetVersionFun;
+
+			auto get_version = reinterpret_cast<GetVersionFun>(dlsym(parSOHandle, "dindexer_backend_iface_version"));
+			return get_version();
+		}
+	} //unnamed namespace
+
+	std::string backend_name (const std::string& parSOPath) {
+		assert(not parSOPath.empty());
+		using SoHandle = std::unique_ptr<void, int(*)(void*)>;
+
+		auto handle = SoHandle(dlopen(parSOPath.c_str(), RTLD_LAZY), &dlclose);
+		if (handle)
+			return backend_name(handle.get());
+		else
+			return std::string();
+	}
+
+	BackendPlugin::BackendPlugin() :
+		m_lib(nullptr, &dlclose),
+		m_backend(nullptr, &nop_destroy),
+		m_name()
+	{
+	}
+
+	BackendPlugin::BackendPlugin (const std::string& parSOPath, const YAML::Node* parConfig) :
+		m_lib(dlopen(parSOPath.c_str(), RTLD_LAZY), &dlclose),
+		m_backend(load_backend(m_lib.get(), parConfig)),
+		m_name(backend_name(m_lib.get())),
+		m_iface_ver(backend_iface_version(m_lib.get()))
+	{
+		if (g_current_iface_version != m_iface_ver) {
+			m_backend.reset();
+			m_lib.reset();
+		}
+	}
+
+	BackendPlugin::~BackendPlugin() noexcept {
+	}
+
+	const boost::string_ref& BackendPlugin::name() const {
+		return m_name;
+	}
+
+	Backend& BackendPlugin::backend() {
+		assert(m_lib);
+		assert(m_backend);
+		if (not m_backend) {
+			throw std::bad_function_call();
+		}
+
+		return *m_backend;
+	}
+
+	const Backend& BackendPlugin::backend() const {
+		assert(m_lib);
+		assert(m_backend);
+		if (not m_backend) {
+			throw std::bad_function_call();
+		}
+
+		return *m_backend;
+	}
+
+	bool BackendPlugin::is_loaded() const {
+		return static_cast<bool>(m_backend);
+	}
+
+	int BackendPlugin::backend_interface_version() const {
+		return m_iface_ver;
+	}
+
+	int BackendPlugin::max_supported_interface_version() const {
+		return g_current_iface_version;
+	}
+} //namespace dindb

src/backends/postgresql/CMakeLists.txt

@@ -0,0 +1,26 @@
+project(${bare_name}-backend-postgresql CXX)
+
+add_library(${PROJECT_NAME} SHARED
+	tag.cpp
+	delete.cpp
+	locate.cpp
+	scan.cpp
+	navigate.cpp
+	backend_postgresql.cpp
+)
+
+target_include_directories(${PROJECT_NAME} SYSTEM
+	PUBLIC ${Boost_INCLUDE_DIRS}
+)
+
+target_link_libraries(${PROJECT_NAME}
+	PRIVATE ${bare_name}-inc
+	PRIVATE ${bare_name}-pq
+)
+
+install(TARGETS ${PROJECT_NAME}
+	LIBRARY DESTINATION lib
+	RUNTIME DESTINATION bin
+	ARCHIVE DESTINATION lib/static
+)
+ln_backend(${PROJECT_NAME})

src/backends/postgresql/backend_postgresql.cpp

@@ -0,0 +1,192 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "backend_postgresql.hpp"
+#include "backends/exposed_functions.hpp"
+#include "backends/backend_version.hpp"
+#include "tag.hpp"
+#include "delete.hpp"
+#include "scan.hpp"
+#include "locate.hpp"
+#include "navigate.hpp"
+#include "pq/connection.hpp"
+#include <ciso646>
+#include <utility>
+#include <cassert>
+#include <yaml-cpp/yaml.h>
+
+namespace dindb {
+	namespace {
+		struct PostgreConnectionSettings {
+			std::string address;
+			std::string username;
+			std::string password;
+			std::string dbname;
+			uint16_t port;
+		};
+	} //unnamed namespace
+} //namespace dindb
+
+namespace YAML {
+	template<>
+	struct convert<dindb::PostgreConnectionSettings> {
+		static Node encode (const dindb::PostgreConnectionSettings& parSettings) {
+			Node node;
+			node["address"] = parSettings.address;
+			node["username"] = parSettings.username;
+			node["password"] = parSettings.password;
+			node["port"] = parSettings.port;
+			node["dbname"] = parSettings.dbname;
+			return node;
+		}
+
+		static bool decode (const Node& parNode, dindb::PostgreConnectionSettings& parSettings) {
+			if (not parNode.IsMap() or parNode.size() != 5) {
+				return false;
+			}
+
+			parSettings.address = parNode["address"].as<std::string>();
+			parSettings.username = parNode["username"].as<std::string>();
+			parSettings.password = parNode["password"].as<std::string>();
+			parSettings.dbname = parNode["dbname"].as<std::string>();
+			parSettings.port = parNode["port"].as<uint16_t>();
+			return true;
+		}
+	};
+} //namespace YAML
+
+namespace dindb {
+	BackendPostgreSql::BackendPostgreSql (std::string&& parUser, std::string&& parPass, std::string&& parDB, std::string&& parAddr, uint16_t parPort) :
+		m_conn(new pq::Connection(std::move(parUser), std::move(parPass), std::move(parDB), std::move(parAddr), parPort))
+	{
+		assert(m_conn);
+		m_conn->connect();
+	}
+
+	BackendPostgreSql::~BackendPostgreSql() noexcept {
+		m_conn->disconnect();
+	}
+
+	void BackendPostgreSql::connect() {
+		m_conn->connect();
+	}
+
+	void BackendPostgreSql::disconnect() {
+		m_conn->disconnect();
+	}
+
+	void BackendPostgreSql::tag_files (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::tag_files(*m_conn, parFiles, parTags, parSet);
+	}
+
+	void BackendPostgreSql::tag_files (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::tag_files(*m_conn, parRegexes, parTags, parSet);
+	}
+
+	void BackendPostgreSql::delete_tags (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::delete_tags(*m_conn, parFiles, parTags, parSet);
+	}
+
+	void BackendPostgreSql::delete_tags (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::delete_tags(*m_conn, parRegexes, parTags, parSet);
+	}
+
+	void BackendPostgreSql::delete_all_tags (const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
+		dindb::delete_all_tags(*m_conn, parFiles, parSet);
+	}
+
+	void BackendPostgreSql::delete_all_tags (const std::vector<std::string>& parRegexes, GroupIDType parSet) {
+		dindb::delete_all_tags(*m_conn, parRegexes, parSet);
+	}
+
+	void BackendPostgreSql::delete_group (const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
+		dindb::delete_group_from_db(*m_conn, parIDs, parConf);
+	}
+
+	void BackendPostgreSql::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
+		dindb::write_to_db(*m_conn, parData, parSetData, parSignature);
+	}
+
+	bool BackendPostgreSql::search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
+		return dindb::read_from_db(parItem, parSet, *m_conn, parHash);
+	}
+
+	std::vector<LocatedItem> BackendPostgreSql::locate_in_db (const std::string& parSearch, const TagList& parTags) {
+		return dindb::locate_in_db(*m_conn, parSearch, parTags);
+	}
+
+	std::vector<LocatedItem> BackendPostgreSql::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) {
+		return dindb::locate_in_db(*m_conn, parSearch, parTags);
+	}
+
+	std::vector<LocatedSet> BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, bool parCaseInsensitive) {
+		return dindb::locate_sets_in_db(*m_conn, parSearch, parCaseInsensitive);
+	}
+
+	std::vector<LocatedSet> BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
+		return dindb::locate_sets_in_db(*m_conn, parSearch, parSets, parCaseInsensitive);
+	}
+
+	std::vector<GroupIDType> BackendPostgreSql::find_all_sets() {
+		return dindb::find_all_sets(*m_conn);
+	}
+
+	std::vector<dhandy::MaxSizedArray<std::string, 4>> BackendPostgreSql::find_set_details (const std::vector<GroupIDType>& parSets) {
+		return dindb::find_set_details<
+			dindb::SetDetail_ID,
+			dindb::SetDetail_Desc,
+			dindb::SetDetail_CreationDate,
+			dindb::SetDetail_DiskLabel
+		>(*m_conn, parSets);
+	}
+
+	std::vector<dhandy::MaxSizedArray<std::string, 1>> BackendPostgreSql::find_file_details (GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) {
+		return dindb::find_file_details<dindb::FileDetail_Path>(*m_conn, parSetID, parLevel, parDir);
+	}
+
+	std::vector<std::string> BackendPostgreSql::find_paths_starting_by (GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) {
+		return dindb::find_paths_starting_by(*m_conn, parGroupID, parLevel, parPath);
+	}
+} //namespace dindb
+
+extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {
+	if (not parConfig)
+		return nullptr;
+
+	auto& config_node = *parConfig;
+	auto config = config_node["connection"].as<dindb::PostgreConnectionSettings>();
+	return new dindb::BackendPostgreSql(
+		std::move(config.username),
+		std::move(config.password),
+		std::move(config.dbname),
+		std::move(config.address),
+		config.port
+	);
+}
+
+extern "C" void dindexer_destroy_backend (dindb::Backend* parDele) {
+	if (parDele)
+		delete parDele;
+}
+
+extern "C" const char* dindexer_backend_name() {
+	return "postgresql";
+}
+
+extern "C" int dindexer_backend_iface_version() {
+	return dindb::g_current_iface_version;
+}

src/backends/postgresql/backend_postgresql.hpp

@@ -0,0 +1,67 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idBE5BBAFEC7334F39AC9338F193703341
+#define idBE5BBAFEC7334F39AC9338F193703341
+
+#include "backends/db_backend.hpp"
+#include <string>
+#include <cstdint>
+#include <memory>
+
+namespace pq {
+	class Connection;
+} //namespace pq
+
+namespace dindb {
+	class BackendPostgreSql : public Backend {
+	public:
+		BackendPostgreSql ( BackendPostgreSql&& ) = default;
+		BackendPostgreSql ( std::string&& parUser, std::string&& parPass, std::string&& parDB, std::string&& parAddr, uint16_t parPort );
+		virtual ~BackendPostgreSql ( void ) noexcept;
+
+		virtual void connect ( void ) override;
+		virtual void disconnect ( void ) override;
+
+		virtual void tag_files ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void tag_files ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_tags ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_tags ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_all_tags ( const std::vector<FileIDType>& parFiles, GroupIDType parSet ) override;
+		virtual void delete_all_tags ( const std::vector<std::string>& parRegexes, GroupIDType parSet ) override;
+
+		virtual void delete_group ( const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf ) override;
+
+		virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) override;
+		virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) override;
+
+		virtual std::vector<LocatedItem> locate_in_db ( const std::string& parSearch, const TagList& parTags ) override;
+		virtual std::vector<LocatedItem> locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) override;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive ) override;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive ) override;
+
+		virtual std::vector<GroupIDType> find_all_sets ( void ) override;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 4>> find_set_details ( const std::vector<GroupIDType>& parSets ) override;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 1>> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) override;
+		virtual std::vector<std::string> find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) override;
+
+	private:
+		std::unique_ptr<pq::Connection> m_conn;
+	};
+} //namespace dindb
+
+#endif

src/backends/postgresql/delete.cpp

@@ -15,10 +15,9 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "postgre_delete.hpp"
+#include "delete.hpp"
 #include "pq/connection.hpp"
-#include "dindexer-common/settings.hpp"
-#include "helpers/infix_iterator.hpp"
+#include "duckhandy/infix_iterator.hpp"
 #include <sstream>
 #include <utility>
 #include <iterator>
@@ -26,10 +25,11 @@
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>
 #include <boost/lexical_cast.hpp>
+#include <cassert>
 
-namespace din {
+namespace dindb {
 	namespace {
-		IDDescMap fetch_existing_ids (pq::Connection& parConn, const std::vector<uint32_t>& parIDs) {
+		IDDescMap fetch_existing_ids (pq::Connection& parConn, const std::vector<GroupIDType>& parIDs) {
 			using boost::lexical_cast;
 
 			IDDescMap retmap;
@@ -39,7 +39,7 @@ namespace din {
 
 			std::ostringstream oss;
 			oss << "SELECT \"id\",\"desc\" FROM \"sets\" WHERE \"id\"=";
-			boost::copy(parIDs, infix_ostream_iterator<uint32_t>(oss, " OR \"id\"="));
+			boost::copy(parIDs, infix_ostream_iterator<GroupIDType>(oss, " OR \"id\"="));
 			oss << ';';
 
 			auto resultset = parConn.query(oss.str());
@@ -50,10 +50,10 @@ namespace din {
 		}
 	} //unnamed namespace
 
-	void delete_group_from_db (const dinlib::SettingsDB& parDB, const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
-		const auto dele_ids = fetch_existing_ids(conn, parIDs);
+	void delete_group_from_db (pq::Connection& parDB, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+		assert(parDB.is_connected());
+
+		const auto dele_ids = fetch_existing_ids(parDB, parIDs);
 		if (dele_ids.empty()) {
 			return;
 		}
@@ -65,11 +65,11 @@ namespace din {
 		ids.reserve(dele_ids.size());
 		std::ostringstream oss;
 		oss << "BEGIN;\nDELETE FROM \"files\" WHERE \"group_id\"=";
-		boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator<uint32_t>(oss, " OR \"group_id\"="));
+		boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator<GroupIDType>(oss, " OR \"group_id\"="));
 		oss << ";\nDELETE FROM \"sets\" WHERE \"id\"=";
-		boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator<uint32_t>(oss, " OR \"id\"="));
+		boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator<GroupIDType>(oss, " OR \"id\"="));
 		oss << ";\nCOMMIT;";
 
-		conn.query(oss.str());
+		parDB.query(oss.str());
 	}
-} //namespace din
+} //namespace dindb

src/backends/postgresql/delete.hpp

@@ -18,21 +18,21 @@
 #ifndef idB070B86E0E4047B1AF4144DEF2759F3C
 #define idB070B86E0E4047B1AF4144DEF2759F3C
 
+#include "backends/db_backend.hpp"
 #include <functional>
 #include <vector>
 #include <string>
-#include <cstdint>
 #include <map>
 
-namespace dinlib {
-	struct SettingsDB;
-} //namespace dinlib
+namespace pq {
+	class Connection;
+} //namespace pq
 
-namespace din {
-	using IDDescMap = std::map<uint32_t, std::string>;
+namespace dindb {
+	using IDDescMap = std::map<GroupIDType, std::string>;
 	using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
 
-	void delete_group_from_db ( const dinlib::SettingsDB& parDB, const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf );
-} //namespace din
+	void delete_group_from_db ( pq::Connection& parDB, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf );
+} //namespace dindb
 
 #endif

src/backends/postgresql/flatinsertin2dlist.hpp

@@ -18,13 +18,21 @@
 #ifndef idAA901DA47E234E37B325B3192EF50423
 #define idAA901DA47E234E37B325B3192EF50423
 
+//Use this function to turn a 1D sequence of elements that are logically grouped
+//together by position into a 2D list. For example, if you have an input sequence
+//like {A1, B1, C1, A2, B2, C2, A3, B3, C3} and you want to turn it into
+//{ {A1, B1, C1}, {A2, B2, C2}, {A3, B3, C3} } you can use these functions to
+//achieve that. You need to specify the size of the innermost groups, 3 in the
+//example just given, and push_back() will automatically move to the next
+//sublist every 3 items.
+
 #include <cstddef>
 #include <cassert>
 #include <ciso646>
 #include <stdexcept>
 #include <utility>
 
-namespace din {
+namespace dhandy {
 	template <typename OuterList, typename InnerList=typename OuterList::value_type, typename InnerVal=typename InnerList::value_type>
 	class FlatInsertIn2DList {
 	public:
@@ -44,7 +52,7 @@ namespace din {
 		const std::size_t m_inner_count;
 		const std::size_t m_outer_count;
 	};
-} //namespace din
+} //namespace dhandy
 
 #include "flatinsertin2dlist.inl"
 

src/backends/postgresql/flatinsertin2dlist.inl

@@ -15,7 +15,7 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-namespace din {
+namespace dhandy {
 	template <typename OuterList, typename InnerList, typename InnerVal>
 	FlatInsertIn2DList<OuterList, InnerList, InnerVal>::FlatInsertIn2DList (list_type* parList, std::size_t parInnerCount, std::size_t parOuterCount) :
 		m_list(parList),
@@ -70,4 +70,4 @@ namespace din {
 			return (m_list->size() - 1) * m_inner_count + m_list[m_list->size() - 1].size();
 		}
 	}
-} //namespace din
+} //namespace dhandy

src/backends/postgresql/locate.cpp

@@ -15,26 +15,19 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "postgre_locate.hpp"
+#include "locate.hpp"
+#include "dindexer-machinery/recorddata.hpp"
 #include "pq/connection.hpp"
+#include "query_count_limit.hpp"
 #include "dindexer-machinery/tiger.hpp"
 #include <utility>
 #include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
+#include <cassert>
 
-namespace din {
+namespace dindb {
 	namespace {
-		const int g_max_results = 200;
-
-		pq::Connection make_pq_conn ( const dinlib::SettingsDB& parDB, bool parOpen=true );
-
-		pq::Connection make_pq_conn (const dinlib::SettingsDB& parDB, bool parOpen) {
-			auto conn = pq::Connection(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-			if (parOpen) {
-				conn.connect();
-			}
-			return conn;
-		}
+		const int g_max_results = g_query_count_limit;
 
 		std::vector<LocatedSet> sets_result_to_vec (pq::ResultSet&& parResult) {
 			using boost::lexical_cast;
@@ -94,21 +87,21 @@ namespace din {
 		}
 	} //unnamed namespace
 
-	std::vector<LocatedItem> locate_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const TagList& parTags) {
-		auto conn = make_pq_conn(parDB);
+	std::vector<LocatedItem> locate_in_db (pq::Connection& parDB, const std::string& parSearch, const TagList& parTags) {
+		assert(parDB.is_connected());
 
 		const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"path\" ~ $1";
-		return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch);
+		return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch);
 	}
 
-	std::vector<LocatedItem> locate_in_db (const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
-		auto conn = make_pq_conn(parDB);
+	std::vector<LocatedItem> locate_in_db (pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
+		assert(parDB.is_connected());
 		const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"hash\"=$1";
-		return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true));
+		return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true));
 	}
 
-	std::vector<LocatedSet> locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, bool parCaseInsensitive) {
-		auto conn = make_pq_conn(parDB);
+	std::vector<LocatedSet> locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive) {
+		assert(parDB.is_connected());
 
 		const std::string query = std::string("SELECT \"id\", \"desc\", "
 			"(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", "
@@ -116,16 +109,16 @@ namespace din {
 			"FROM \"sets\" WHERE str_match_partial(\"desc\", $1, $2) LIMIT "
 		) + std::to_string(g_max_results) + ";";
 
-		auto result = conn.query(query, parSearch, parCaseInsensitive);
+		auto result = parDB.query(query, parSearch, parCaseInsensitive);
 		return sets_result_to_vec(std::move(result));
 	}
 
-	std::vector<LocatedSet> locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive) {
+	std::vector<LocatedSet> locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
 		if (parSets.empty()) {
 			return locate_sets_in_db(parDB, parSearch, parCaseInsensitive);
 		}
 
-		auto conn = make_pq_conn(parDB);
+		assert(parDB.is_connected());
 
 		const std::string query = std::string("SELECT \"id\", \"desc\", "
 			"(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", "
@@ -133,7 +126,7 @@ namespace din {
 			"FROM \"sets\" WHERE \"id\" = ANY($1) AND str_match_partial(\"desc\", $3, $2) LIMIT "
 		) + std::to_string(g_max_results) + ";";
 
-		auto result = conn.query(query, parSearch, parCaseInsensitive, parSets);
+		auto result = parDB.query(query, parSearch, parCaseInsensitive, parSets);
 		return sets_result_to_vec(std::move(result));
 	}
-} //namespace din
+} //namespace dindb

src/backends/postgresql/locate.hpp

@@ -0,0 +1,37 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id1AE05A59AE0E4A4490040FD85D9AF665
+#define id1AE05A59AE0E4A4490040FD85D9AF665
+
+#include <vector>
+#include <string>
+#include <cstdint>
+#include "backends/db_backend.hpp"
+
+namespace pq {
+	class Connection;
+} //namespace pq
+
+namespace dindb {
+	std::vector<LocatedItem> locate_in_db ( pq::Connection& parDB, const std::string& parSearch, const TagList& parTags );
+	std::vector<LocatedItem> locate_in_db ( pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags );
+	std::vector<LocatedSet> locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive );
+	std::vector<LocatedSet> locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive );
+} //namespace dindb
+
+#endif

src/backends/postgresql/navigate.cpp

@@ -0,0 +1,155 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "navigate.hpp"
+#include "pq/connection.hpp"
+#include "duckhandy/infix_iterator.hpp"
+#include "query_count_limit.hpp"
+#include <ciso646>
+#include <utility>
+#include <boost/lexical_cast.hpp>
+#include <sstream>
+#include <boost/range/algorithm/copy.hpp>
+#include <cassert>
+
+namespace dindb {
+	namespace {
+		const uint32_t g_files_query_limit = g_query_count_limit;
+
+		std::ostream& operator<< (std::ostream& parOut, const std::vector<std::string>& parCols) {
+			parOut << '"';
+			boost::copy(parCols, infix_ostream_iterator<std::string>(parOut, "\", \""));
+			parOut << '"';
+			return parOut;
+		}
+	} //unnamed namespace
+
+	namespace implem {
+		const SetDetailsMap g_set_details_map {
+			{SetDetail_Desc, "desc"},
+			{SetDetail_Type, "type"},
+			{SetDetail_CreationDate, "creation"},
+			{SetDetail_AppName, "app_name"},
+			{SetDetail_ID, "id"},
+			{SetDetail_DiskLabel, "disk_label"},
+			{SetDetail_FSUuid, "fs_uuid"}
+		};
+		const FileDetailsMap g_file_details_map {
+			{FileDetail_ID, "id"},
+			{FileDetail_Path, "path"},
+			{FileDetail_Level, "level"},
+			{FileDetail_GroupID, "group_id"},
+			{FileDetail_IsDir, "is_directory"},
+			{FileDetail_IsSymLink, "is_symlink"},
+			{FileDetail_Size, "size"},
+			{FileDetail_Hash, "hash"},
+			{FileDetail_IsHashValid, "is_hash_valid"},
+			{FileDetail_ATime, "access_time"},
+			{FileDetail_MTime, "modify_time"},
+			{FileDetail_Unreadable, "unreadable"},
+			{FileDetail_MimeType, "mimetype"},
+			{FileDetail_Charset, "charset"}
+		};
+
+		void query_no_conditions (pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parTable, const std::vector<GroupIDType>& parIDs, std::function<void(std::string&&)> parCallback) {
+			assert(parDB.is_connected());
+
+			std::ostringstream oss;
+			oss << "SELECT " << parCols << ' ' <<
+				"FROM \"" << parTable << "\" " <<
+				"WHERE \"id\"=ANY($1) " <<
+				"ORDER BY \"desc\" ASC " <<
+				"LIMIT " << g_files_query_limit << ';';
+
+			auto result = parDB.query(oss.str(), parIDs);
+			for (auto row : result) {
+				for (auto val : row) {
+					parCallback(std::move(val));
+				}
+			}
+		}
+
+		void query_files_in_dir (pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, GroupIDType parGroupID, QueryCallback parCallback) {
+			assert(parDB.is_connected());
+
+			std::ostringstream oss;
+			oss << "SELECT " << parCols << ' ' <<
+				"FROM \"files\" WHERE " <<
+				"\"level\"=$1 " <<
+				"AND \"group_id\"=$2 " <<
+				"AND str_begins_with(\"path\", COALESCE($3, '')) " <<
+				"ORDER BY \"is_directory\" DESC, \"path\" ASC " <<
+				"LIMIT " << g_files_query_limit << ';';
+
+			auto result = parDB.query(
+				oss.str(),
+				parLevel,
+				parGroupID,
+				parDir
+			);
+			for (auto row : result) {
+				for (auto val : row) {
+					parCallback(std::move(val));
+				}
+			}
+		}
+	} //namespace implem
+
+	std::vector<GroupIDType> find_all_sets (pq::Connection& parDB) {
+		using boost::lexical_cast;
+
+		assert(parDB.is_connected());
+
+		const std::string query = "SELECT \"id\" FROM \"sets\";";
+		auto res = parDB.query(query);
+		std::vector<GroupIDType> retval;
+
+		retval.reserve(res.size());
+		for (const auto& row : res) {
+			retval.push_back(lexical_cast<GroupIDType>(row[0]));
+		}
+		return retval;
+	}
+
+	std::vector<std::string> find_paths_starting_by (pq::Connection& parDB, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) {
+		assert(parDB.is_connected());
+
+		std::ostringstream oss;
+		oss << "SELECT \"path\" ||\n" <<
+			"(SELECT CASE \"is_directory\"\n" <<
+			"WHEN TRUE THEN '/'\n" <<
+			"ELSE ''\n" <<
+			"END) as path FROM \"files\" WHERE \"group_id\"=$1 AND " <<
+			"\"level\"=$2 AND str_begins_with(\"path\", COALESCE($3, '')) " <<
+			"ORDER BY \"is_directory\" DESC, \"path\" ASC LIMIT " <<
+			g_files_query_limit << ';';
+
+		auto result = parDB.query(
+			oss.str(),
+			parGroupID,
+			parLevel,
+			parPath
+		);
+		std::vector<std::string> retval;
+		retval.reserve(retval.size());
+		for (auto row : result) {
+			assert(not row.empty());
+			retval.push_back(row[0]);
+		}
+		return retval;
+	}
+} //namespace dindb

src/backends/postgresql/navigate.hpp

@@ -20,7 +20,8 @@
 
 #include "dindexer-machinery/recorddata.hpp"
 #include "flatinsertin2dlist.hpp"
-#include "MaxSizedArray.hpp"
+#include "duckhandy/MaxSizedArray.hpp"
+#include "backends/db_backend.hpp"
 #include <memory>
 #include <cstdint>
 #include <vector>
@@ -31,21 +32,21 @@
 #include <functional>
 #include <boost/utility/string_ref.hpp>
 
-namespace dinlib {
-	struct SettingsDB;
-} //namespace dinlib
-
 namespace pq {
 	class Connection;
 } //namespace pq
 
-namespace din {
+namespace dindb {
+	using dhandy::MaxSizedArray;
+
 	enum SetDetails {
 		SetDetail_Desc = 0x01,
 		SetDetail_Type = 0x02,
-		SetDetail_CreeationDate = 0x04,
+		SetDetail_CreationDate = 0x04,
 		SetDetail_AppName = 0x08,
-		SetDetail_ID = 0x10
+		SetDetail_ID = 0x10,
+		SetDetail_DiskLabel = 0x20,
+		SetDetail_FSUuid = 0x40
 	};
 
 	enum FileDetails {
@@ -65,40 +66,25 @@ namespace din {
 		FileDetail_Charset = 0x2000
 	};
 
-	class DBSource {
-	public:
-		explicit DBSource ( const dinlib::SettingsDB& parDBSettings );
-		~DBSource ( void ) noexcept;
+	std::vector<GroupIDType> find_all_sets ( pq::Connection& parDB );
 
-		void disconnect ( void );
-		std::vector<uint32_t> sets ( void );
+	template <SetDetails... D>
+	auto find_set_details ( pq::Connection& parDB, const std::vector<GroupIDType>& parIDs ) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>>;
 
-		template <SetDetails... D>
-		auto set_details ( const std::vector<uint32_t>& parIDs ) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>>;
+	template <FileDetails... D>
+	auto find_file_details ( pq::Connection& parDB, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>>;
 
-		template <FileDetails... D>
-		auto file_details ( uint32_t parSetID, uint16_t parLevel, boost::string_ref parDir ) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>>;
-
-		std::vector<std::string> paths_starting_by ( uint32_t parGroupID, uint16_t parLevel, boost::string_ref parPath );
-
-	private:
-		struct LocalData;
-		typedef std::map<SetDetails, std::string> SetDetailsMap;
-		typedef std::map<FileDetails, std::string> FileDetailsMap;
-		typedef std::vector<std::string> ColumnList;
-		typedef std::function<void(std::string&&)> QueryCallback;
-
-		pq::Connection& get_conn ( void );
-		void query_no_conditions ( const ColumnList& parCols, boost::string_ref parTable, const std::vector<uint32_t>& parIDs, QueryCallback parCallback );
-		void query_files_in_dir ( const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, uint32_t parGroupID, QueryCallback parCallback );
-
-		static const SetDetailsMap m_set_details_map;
-		static const FileDetailsMap m_file_details_map;
-
-		std::unique_ptr<LocalData> m_local_data;
-	};
+	std::vector<std::string> find_paths_starting_by ( pq::Connection& parDB, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath );
 
 	namespace implem {
+		typedef std::vector<std::string> ColumnList;
+		typedef std::function<void(std::string&&)> QueryCallback;
+		typedef std::map<SetDetails, std::string> SetDetailsMap;
+		typedef std::map<FileDetails, std::string> FileDetailsMap;
+
+		extern const SetDetailsMap g_set_details_map;
+		extern const FileDetailsMap g_file_details_map;
+
 		template <class M, M... Details>
 		inline
 		std::vector<std::string> make_columns_vec (const std::map<M, std::string>& parDic) {
@@ -115,35 +101,42 @@ namespace din {
 			}
 			return columns;
 		}
+
+		void query_no_conditions ( pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parTable, const std::vector<GroupIDType>& parIDs, QueryCallback parCallback );
+		void query_files_in_dir ( pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, GroupIDType parGroupID, QueryCallback parCallback );
 	} //namespace implem
 
 	template <SetDetails... D>
-	auto DBSource::set_details (const std::vector<uint32_t>& parIDs) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>> {
+	inline
+	auto find_set_details (pq::Connection& parDB, const std::vector<GroupIDType>& parIDs) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>> {
+		using dhandy::FlatInsertIn2DList;
 		typedef std::vector<MaxSizedArray<std::string, sizeof...(D)>> ReturnType;
-		typedef void(din::FlatInsertIn2DList<ReturnType>::*FlatPushBackFunc)(std::string&&);
+		typedef void(FlatInsertIn2DList<ReturnType>::*FlatPushBackFunc)(std::string&&);
 
-		const auto columns = implem::make_columns_vec<SetDetails, D...>(m_set_details_map);
+		const auto columns = implem::make_columns_vec<SetDetails, D...>(implem::g_set_details_map);
 
 		ReturnType list;
 		FlatInsertIn2DList<ReturnType> flat_list(&list, sizeof...(D));
 		FlatPushBackFunc pback_func = &FlatInsertIn2DList<ReturnType>::push_back;
-		this->query_no_conditions(columns, "sets", parIDs, std::bind(pback_func, &flat_list, std::placeholders::_1));
+		implem::query_no_conditions(parDB, columns, "sets", parIDs, std::bind(pback_func, &flat_list, std::placeholders::_1));
 		return list;
 	}
 
 	template <FileDetails... D>
-	auto DBSource::file_details (uint32_t parSetID, uint16_t parLevel, boost::string_ref parDir) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>> {
+	inline
+	auto find_file_details (pq::Connection& parDB, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) -> std::vector<MaxSizedArray<std::string, sizeof...(D)>> {
+		using dhandy::FlatInsertIn2DList;
 		typedef std::vector<MaxSizedArray<std::string, sizeof...(D)>> ReturnType;
-		typedef void(din::FlatInsertIn2DList<ReturnType>::*FlatPushBackFunc)(std::string&&);
+		typedef void(FlatInsertIn2DList<ReturnType>::*FlatPushBackFunc)(std::string&&);
 
-		const auto columns = implem::make_columns_vec<FileDetails, D...>(m_file_details_map);
+		const auto columns = implem::make_columns_vec<FileDetails, D...>(implem::g_file_details_map);
 
 		ReturnType list;
 		FlatInsertIn2DList<ReturnType> flat_list(&list, sizeof...(D));
 		FlatPushBackFunc pback_func = &FlatInsertIn2DList<ReturnType>::push_back;
-		this->query_files_in_dir(columns, parDir, parLevel, parSetID, std::bind(pback_func, &flat_list, std::placeholders::_1));
+		implem::query_files_in_dir(parDB, columns, parDir, parLevel, parSetID, std::bind(pback_func, &flat_list, std::placeholders::_1));
 		return list;
 	}
-} //namespace din
+} //namespace dindb
 
 #endif

src/backends/postgresql/query_count_limit.hpp

@@ -0,0 +1,25 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id367C9335A3444BC0846EF4E2A27AB9A9
+#define id367C9335A3444BC0846EF4E2A27AB9A9
+
+namespace dindb {
+	constexpr const uint32_t g_query_count_limit = 500;
+};
+
+#endif

src/backends/postgresql/scan.cpp

@@ -15,9 +15,8 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dbbackend.hpp"
+#include "scan.hpp"
 #include "pq/connection.hpp"
-#include "dindexer-common/settings.hpp"
 #include "dindexer-machinery/recorddata.hpp"
 #include <string>
 #include <sstream>
@@ -27,20 +26,20 @@
 #include <memory>
 #include <boost/utility/string_ref.hpp>
 #include <chrono>
+#include <cassert>
 
-namespace din {
+namespace dindb {
 	namespace {
 	} //unnamed namespace
 
-	bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parHash) {
+	bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash) {
 		using boost::lexical_cast;
 
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+		assert(parDB.is_connected());
 
 		uint32_t group_id;
 		{
-			auto resultset = conn.query(
+			auto resultset = parDB.query(
 				"SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash=$1 LIMIT 1;",
 				tiger_to_string(parHash, true)
 			);
@@ -64,8 +63,8 @@ namespace din {
 		}
 
 		{
-			auto resultset = conn.query(
-				"SELECT \"desc\",\"type\",\"disk_number\",\"content_type\" FROM sets WHERE \"id\"=$1;",
+			auto resultset = parDB.query(
+				"SELECT \"desc\",\"type\",\"disk_number\",\"fs_uuid\",\"disk_label\",\"content_type\" FROM sets WHERE \"id\"=$1;",
 				group_id
 			);
 			if (resultset.empty()) {
@@ -78,33 +77,36 @@ namespace din {
 			parSet.type = lexical_cast<char>(row["type"]);
 			parSet.name = row["desc"];
 			parSet.disk_number = lexical_cast<uint32_t>(row["disk_number"]);
+			parSet.fs_uuid = row["fs_uuid"];
+			parSet.disk_label = row["disk_label"];
 			parSet.content_type = lexical_cast<char>(row["content_type"]);
 		}
 		return true;
 	}
 
-	void write_to_db (const dinlib::SettingsDB& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
+	void write_to_db (pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
 		using std::chrono::system_clock;
 		using boost::lexical_cast;
 
+		assert(parDB.is_connected());
+
 		if (parData.empty()) {
 			return;
 		}
 
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
-
-		conn.query("BEGIN;");
+		parDB.query("BEGIN;");
 		uint32_t new_group_id;
 		{
-			auto id_res = conn.query("INSERT INTO \"sets\" "
+			auto id_res = parDB.query("INSERT INTO \"sets\" "
 				"(\"desc\",\"type\",\"app_name\""
-				",\"content_type\") "
-				"VALUES ($1, $2, $3, $4) RETURNING \"id\";",
+				",\"content_type\",\"fs_uuid\",\"disk_label\") "
+				"VALUES ($1, $2, $3, $4, $5, $6) RETURNING \"id\";",
 				parSetData.name,
 				boost::string_ref(&parSetData.type, 1),
 				parSignature,
-				boost::string_ref(&parSetData.content_type, 1)
+				boost::string_ref(&parSetData.content_type, 1),
+				parSetData.fs_uuid,
+				parSetData.disk_label
 			);
 			assert(id_res.size() == 1);
 			assert(id_res[0].size() == 1);
@@ -127,7 +129,7 @@ namespace din {
 
 			const auto& itm = parData[z];
 			assert(itm.path().data());
-			conn.query(query,
+			parDB.query(query,
 				(itm.path().empty() ? empty_path_string : itm.path()),
 				tiger_to_string(itm.hash),
 				itm.level,
@@ -143,6 +145,6 @@ namespace din {
 				itm.mime_charset()
 			);
 		}
-		conn.query("COMMIT;");
+		parDB.query("COMMIT;");
 	}
-} //namespace din
+} //namespace dindb

src/backends/postgresql/scan.hpp

@@ -22,20 +22,21 @@
 #include <vector>
 #include <cstdint>
 
-namespace dinlib {
-	struct SettingsDB;;
-} //namespace dinlib
-
 namespace mchlib {
 	struct FileRecordData;
-	struct SetRecordData;
 	struct SetRecordDataFull;
 	struct TigerHash;
 } //namespace mchlib
 
-namespace din {
-	void write_to_db ( const dinlib::SettingsDB& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
-	bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parHash );
-} //namespace din
+namespace pq {
+	class Connection;
+} //namespace pq
+
+namespace dindb {
+	struct Settings;;
+
+	void write_to_db ( pq::Connection& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature );
+	bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash );
+} //namespace dindb
 
 #endif

src/backends/postgresql/tag.cpp

@@ -15,141 +15,135 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tag_postgres.hpp"
+#include "tag.hpp"
 #include "pq/connection.hpp"
-#include "dindexer-common/settings.hpp"
 #include <ciso646>
+#include <cassert>
 
-namespace din {
-	void tag_files (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+namespace dindb {
+	void tag_files (pq::Connection& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
-		if (parSet.is_valid) {
+		if (InvalidGroupID != parSet) {
 			const std::string query =
 				"UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"id\"=ANY($2) AND \"group_id\"=$3;";
-			conn.query(query, parTags, parFiles, parSet.group_id);
+			parDB.query(query, parTags, parFiles, parSet);
 		}
 		else {
 			const std::string query =
 				"UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"id\"=ANY($2);";
-			conn.query(query, parTags, parFiles);
+			parDB.query(query, parTags, parFiles);
 		}
 	}
 
-	void tag_files (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+	void tag_files (pq::Connection& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
-		if (parSet.is_valid) {
+		if (InvalidGroupID != parSet) {
 			if (parRegexes.size() == 1) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"group_id\"=$2 AND \"path\" ~ $3;";
-				conn.query(query, parTags, parSet.group_id, parRegexes.front());
+				parDB.query(query, parTags, parSet, parRegexes.front());
 			}
 			else if (parRegexes.size() > 1) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"group_id\"=$2 AND \"path\" ~ ANY($3);";
-				conn.query(query, parTags, parSet.group_id, parRegexes);
+				parDB.query(query, parTags, parSet, parRegexes);
 			}
 			else if (parRegexes.size() == 0) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) WHERE \"group_id\"=$2 ORDER BY 1);";
-				conn.query(query, parTags, parSet.group_id);
+				parDB.query(query, parTags, parSet);
 			}
 		}
 		else {
 			if (parRegexes.size() == 1) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"path\" ~ $2;";
-				conn.query(query, parTags, parRegexes.front());
+				parDB.query(query, parTags, parRegexes.front());
 			}
 			else if (parRegexes.size() > 1) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"path\" ~ ANY($2);";
-				conn.query(query, parTags, parRegexes);
+				parDB.query(query, parTags, parRegexes);
 			}
 			else if (parRegexes.size() == 0) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1);";
-				conn.query(query, parTags);
+				parDB.query(query, parTags);
 			}
 		}
 	}
 
-	void delete_tags (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+	void delete_tags (pq::Connection& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
 		if (parTags.size() == 1) {
-			if (parSet.is_valid) {
+			if (InvalidGroupID != parSet) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"id\" = ANY($2) AND \"group_id\" = $3;";
-				conn.query(query, parTags.front(), parFiles, parSet.group_id);
+				parDB.query(query, parTags.front(), parFiles, parSet);
 			}
 			else {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"id\" = ANY($2);";
-				conn.query(query, parTags.front(), parFiles);
+				parDB.query(query, parTags.front(), parFiles);
 			}
 		}
 		else {
-			if (parSet.is_valid) {
+			if (InvalidGroupID != parSet) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"id\" = ANY($2) AND \"group_id\" = $3;";
-				conn.query(query, parTags, parFiles, parSet.group_id);
+				parDB.query(query, parTags, parFiles, parSet);
 			}
 			else {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"id\" = ANY($2);";
-				conn.query(query, parTags, parFiles);
+				parDB.query(query, parTags, parFiles);
 			}
 		}
 	}
 
-	void delete_tags (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+	void delete_tags (pq::Connection& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
 		if (parTags.size() == 1) {
-			if (parSet.is_valid) {
+			if (InvalidGroupID != parSet) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE  \"group_id\" = $3 AND \"path\" ~ ANY($3);";
-				conn.query(query, parTags.front(), parSet.group_id, parRegexes);
+				parDB.query(query, parTags.front(), parSet, parRegexes);
 			}
 			else {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"path\" ~ ANY($2);";
-				conn.query(query, parTags.front(), parRegexes);
+				parDB.query(query, parTags.front(), parRegexes);
 			}
 		}
 		else {
-			if (parSet.is_valid) {
+			if (InvalidGroupID != parSet) {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"group_id\" = $2 AND \"path\" ~ ANY($3);";
-				conn.query(query, parTags, parSet.group_id, parRegexes);
+				parDB.query(query, parTags, parSet, parRegexes);
 			}
 			else {
 				const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"path\" = ANY($2);";
-				conn.query(query, parTags, parRegexes);
+				parDB.query(query, parTags, parRegexes);
 			}
 		}
 	}
 
-	void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+	void delete_all_tags (pq::Connection& parDB, const std::vector<uint64_t>& parFiles, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
-		if (parSet.is_valid) {
+		if (InvalidGroupID != parSet) {
 			const std::string query =
 				"UPDATE \"files\" SET \"tags\" = '{}' WHERE \"id\"=ANY($1) AND \"group_id\"=$2;";
-			conn.query(query, parFiles, parSet.group_id);
+			parDB.query(query, parFiles, parSet);
 		}
 		else {
 			const std::string query =
 				"UPDATE \"files\" SET \"tags\" = '{}' WHERE \"id\"=ANY($1);";
-			conn.query(query, parFiles);
+			parDB.query(query, parFiles);
 		}
 	}
 
-	void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, OwnerSetInfo parSet) {
-		pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
-		conn.connect();
+	void delete_all_tags (pq::Connection& parDB, const std::vector<std::string>& parRegexes, GroupIDType parSet) {
+		assert(parDB.is_connected());
 
-		if (parSet.is_valid) {
+		if (InvalidGroupID != parSet) {
 			const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"group_id\"=$1 AND \"path\" ~ ANY($2);";
-			conn.query(query, parSet.group_id, parRegexes);
+			parDB.query(query, parSet, parRegexes);
 		}
 		else {
 			const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"path\" ~ ANY($2);";
-			conn.query(query, parRegexes);
+			parDB.query(query, parRegexes);
 		}
 	}
-} //namespace din
+} //namespace dindb

src/backends/postgresql/tag.hpp

@@ -0,0 +1,43 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idE1E1650A8CAA4949BD6D4D58BF2599F5
+#define idE1E1650A8CAA4949BD6D4D58BF2599F5
+
+#include "backends/db_backend.hpp"
+#include <vector>
+#include <boost/utility/string_ref.hpp>
+#include <cstdint>
+#include <pq/connection.hpp>
+
+namespace pq {
+	class Connection;
+} //namespace pq
+
+namespace dindb {
+	struct Settings;
+
+	void tag_files ( pq::Connection& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet );
+	void tag_files ( pq::Connection& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet );
+
+	void delete_tags ( pq::Connection& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet );
+	void delete_tags ( pq::Connection& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet );
+	void delete_all_tags ( pq::Connection& parDB, const std::vector<uint64_t>& parFiles, GroupIDType parSet );
+	void delete_all_tags ( pq::Connection& parDB, const std::vector<std::string>& parRegexes, GroupIDType parSet );
+} //namespace dindb
+
+#endif

src/backends/redis/CMakeLists.txt

@@ -0,0 +1,56 @@
+project(${bare_name}-backend-redis CXX)
+
+set(DINDEXER_REDIS_SCRIPTS_PATH "${CMAKE_INSTALL_PREFIX}/${bare_name}/redis" CACHE STRING "Path where Lua scripts for Redis are stored")
+
+find_package(Boost 1.53.0 REQUIRED COMPONENTS regex)
+
+add_library(${PROJECT_NAME} SHARED
+	backend_redis.cpp
+	tag.cpp
+	delete.cpp
+	find.cpp
+)
+
+target_include_directories(${PROJECT_NAME} SYSTEM
+	PUBLIC ${Boost_INCLUDE_DIRS}
+)
+target_include_directories(${PROJECT_NAME}
+	PRIVATE ${CMAKE_CURRENT_BINARY_DIR}
+	PUBLIC ${CMAKE_SOURCE_DIR}/src/incredis
+)
+
+target_link_libraries(${PROJECT_NAME}
+	PRIVATE ${bare_name}-inc
+	PUBLIC ${bare_name}-core
+	PRIVATE ${Boost_LIBRARIES}
+	PRIVATE incredis
+)
+
+configure_file(
+	redisConfig.h.in
+	${CMAKE_CURRENT_BINARY_DIR}/redisConfig.h
+)
+set(LUA_SCRIPTS
+	tag_if_in_set.lua
+	dele_tag_if_in_set.lua
+	dele_hash.lua
+)
+
+install(TARGETS ${PROJECT_NAME}
+	LIBRARY DESTINATION lib
+	RUNTIME DESTINATION bin
+	ARCHIVE DESTINATION lib/static
+)
+
+set(lua_script_list "")
+foreach (lua_script ${LUA_SCRIPTS})
+	get_filename_component(lua_script_basename "${lua_script}" NAME)
+	configure_file("${lua_script}" "${CMAKE_CURRENT_BINARY_DIR}/lua/${lua_script_basename}" COPYONLY)
+	list(APPEND lua_script_list "${CMAKE_CURRENT_BINARY_DIR}/lua/${lua_script_basename}")
+endforeach()
+unset(lua_script)
+unset(lua_script_basename)
+install(FILES ${lua_script_list} DESTINATION "${DINDEXER_REDIS_SCRIPTS_PATH}")
+unset(lua_script_list)
+
+ln_backend(${PROJECT_NAME})

src/backends/redis/backend_redis.cpp

@@ -0,0 +1,322 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "backend_redis.hpp"
+#include "dindexer-machinery/recorddata.hpp"
+#include "backends/exposed_functions.hpp"
+#include "backends/backend_version.hpp"
+#include "duckhandy/lexical_cast.hpp"
+#include "dindexerConfig.h"
+#include "redisConfig.h"
+#include "duckhandy/stringize.h"
+#include "tag.hpp"
+#include "delete.hpp"
+#include "find.hpp"
+#include "record_data_adapt.hpp"
+#include <utility>
+#include <yaml-cpp/yaml.h>
+#include <array>
+#include <cstdint>
+#include <boost/range/empty.hpp>
+#include <fstream>
+#include <ctime>
+
+namespace dindb {
+	namespace {
+		struct RedisConnectionSettings {
+			std::string address;
+			uint16_t port;
+			uint16_t database;
+		};
+
+		std::string read_script (const dincore::SearchPaths& parSearch, const char* parName) {
+			const auto full_path = parSearch.first_hit(boost::string_ref(parName));
+			if (full_path.empty()) {
+				const std::string msg = std::string("Unable to locate and load Lua script \"") + parName + "\" from any of the given search paths";
+				throw std::runtime_error(msg);
+			}
+
+			std::ifstream script(full_path);
+			std::string retval;
+			script.seekg(0, std::ios::end);
+			retval.reserve(script.tellg());
+			script.seekg(0, std::ios::beg);
+			retval.assign(std::istreambuf_iterator<char>(script), std::istreambuf_iterator<char>());
+
+			return retval;
+		}
+	} //unnamed namespace
+} //namespace dindb
+
+namespace YAML {
+	template<>
+	struct convert<dindb::RedisConnectionSettings> {
+		static Node encode (const dindb::RedisConnectionSettings& parSettings) {
+			Node node;
+			node["address"] = parSettings.address;
+			node["port"] = parSettings.port;
+			node["database"] = parSettings.database;
+			return node;
+		}
+
+		static bool decode (const Node& parNode, dindb::RedisConnectionSettings& parSettings) {
+			if (not parNode.IsMap() or parNode.size() != 2) {
+				return false;
+			}
+
+			parSettings.address = parNode["address"].as<std::string>();
+			parSettings.port = parNode["port"].as<uint16_t>();
+			if (parNode["database"])
+				parSettings.database = parNode["database"].as<uint16_t>();
+			else
+				parSettings.database = 0;
+			return true;
+		}
+	};
+} //namespace YAML
+
+namespace dindb {
+	BackendRedis::BackendRedis(std::string&& parAddress, uint16_t parPort, uint16_t parDatabase, bool parConnect, dincore::SearchPaths&& parLuaPaths) :
+		m_redis(std::move(parAddress), parPort),
+		m_tag_if_in_set(),
+		m_lua_script_paths(std::move(parLuaPaths)),
+		m_database(parDatabase)
+	{
+		if (parConnect)
+			this->connect();
+	}
+
+	BackendRedis::~BackendRedis() noexcept {
+	}
+
+	void BackendRedis::connect() {
+		using dhandy::lexical_cast;
+
+		m_redis.connect();
+		m_redis.wait_for_connect();
+		if (m_redis.is_connected()) {
+			auto batch = m_redis.make_batch();
+			batch.select(m_database);
+			batch.client_setname(PROGRAM_NAME "_v" STRINGIZE(VERSION_MAJOR) "." STRINGIZE(VERSION_MINOR) "." STRINGIZE(VERSION_PATCH));
+			batch.script_flush();
+			batch.throw_if_failed();
+		}
+		else {
+			std::ostringstream oss;
+			oss << "Error connecting to Redis: " << m_redis.command().connection_error();
+			throw std::runtime_error(oss.str());
+		}
+
+		m_tag_if_in_set = m_redis.command().make_script(read_script(m_lua_script_paths, "tag_if_in_set.lua"));
+		m_dele_tag_if_in_set = m_redis.command().make_script(read_script(m_lua_script_paths, "dele_tag_if_in_set.lua"));
+		m_dele_hash = m_redis.command().make_script(read_script(m_lua_script_paths, "dele_hash.lua"));
+	}
+
+	void BackendRedis::disconnect() {
+		m_redis.disconnect();
+	}
+
+	void BackendRedis::tag_files (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::tag_files(m_redis, m_tag_if_in_set, parFiles, parTags, parSet);
+	}
+
+	void BackendRedis::tag_files (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::tag_files(m_redis, m_tag_if_in_set, parRegexes, parTags, parSet);
+	}
+
+	void BackendRedis::delete_tags (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parFiles, parTags, parSet);
+	}
+
+	void BackendRedis::delete_tags (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parTags, parSet);
+	}
+
+	void BackendRedis::delete_all_tags (const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
+		dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parFiles, parSet);
+	}
+
+	void BackendRedis::delete_all_tags (const std::vector<std::string>& parRegexes, GroupIDType parSet) {
+		dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parSet);
+	}
+
+	void BackendRedis::delete_group (const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+		delete_group_from_db(m_redis, m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf);
+	}
+
+	void BackendRedis::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
+		using dhandy::lexical_cast;
+		using boost::string_ref;
+
+		const auto data_size = static_cast<int>(parData.size());
+		const auto group_id_int = m_redis.hincrby(PROGRAM_NAME ":indices", "set", 1);
+		const auto file_id_int = m_redis.hincrby(PROGRAM_NAME ":indices", "files", data_size);
+
+		const auto group_id = lexical_cast<std::string>(group_id_int);
+		const std::string set_key = PROGRAM_NAME ":set:" + group_id;
+		const std::string level_key = PROGRAM_NAME ":levels:" + group_id;
+		assert(file_id_int >= data_size);
+		const auto base_file_id = file_id_int - data_size + 1;
+
+		auto batch = m_redis.make_batch();
+
+		batch.hmset(
+			set_key,
+			"name", parSetData.name,
+			"disk_label", parSetData.disk_label,
+			"fs_uuid", parSetData.fs_uuid,
+			"type", parSetData.type,
+			"content_type", parSetData.content_type,
+			"base_file_id", lexical_cast<std::string>(base_file_id),
+			"item_count", lexical_cast<std::string>(parData.size()),
+			"dir_count", lexical_cast<std::string>(std::count_if(parData.begin(), parData.end(), [](const mchlib::FileRecordData& r){return r.is_directory;})),
+			"creation", lexical_cast<std::string>(std::time(nullptr)),
+			"app_name", parSignature
+		);
+
+#if !defined(NDEBUG)
+		std::size_t inserted_count = 0;
+#endif
+		for (auto z = base_file_id; z < base_file_id + data_size; ++z) {
+			const std::string file_key = PROGRAM_NAME ":file:" + lexical_cast<std::string>(z);
+			assert(z >= base_file_id);
+			assert(static_cast<std::size_t>(z - base_file_id) < parData.size());
+			const auto& file_data = parData[z - base_file_id];
+			const std::string hash = tiger_to_string(file_data.hash);
+			batch.hmset(
+				file_key,
+				"hash", hash,
+				"path", file_data.path(),
+				"size", lexical_cast<std::string>(file_data.size),
+				"level", lexical_cast<std::string>(file_data.level),
+				"mime_type", file_data.mime_type(),
+				"mime_charset", file_data.mime_charset(),
+				"is_directory", (file_data.is_directory ? '1' : '0'),
+				"is_symlink", (file_data.is_symlink ? '1' : '0'),
+				"unreadable", (file_data.unreadable ? '1' : '0'),
+				"hash_valid", (file_data.hash_valid ? '1' : '0'),
+				"group_id", group_id,
+				"atime", lexical_cast<std::string>(file_data.atime),
+				"mtime", lexical_cast<std::string>(file_data.mtime)
+			);
+
+			batch.sadd(
+				PROGRAM_NAME ":hash:" + hash,
+				lexical_cast<std::string>(z)
+			);
+
+			batch.zadd(level_key, redis::IncRedisBatch::ZADD_None, false, static_cast<double>(file_data.level), file_key);
+#if !defined(NDEBUG)
+			++inserted_count;
+#endif
+		}
+		assert(inserted_count == parData.size());
+
+		batch.throw_if_failed();
+	}
+
+	bool BackendRedis::search_file_by_hash (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
+		using boost::empty;
+
+		const std::string hash_key = PROGRAM_NAME ":hash:" + tiger_to_string(parHash);
+		auto hash_reply = m_redis.srandmember(hash_key);
+		if (not hash_reply) {
+			return false;
+		}
+		else {
+			const auto file_key = PROGRAM_NAME ":file:" + *hash_reply;
+			auto set_key_and_file_item = redis::range_as<FileRecordDataWithGroup>(m_redis.hscan(file_key));
+			parItem = std::move(set_key_and_file_item.second);
+			assert(parItem.hash == parHash);
+			const std::string group_key = PROGRAM_NAME ":set:" + set_key_and_file_item.first;
+
+			auto scan_range = m_redis.hscan(group_key);
+			if (empty(scan_range)) {
+				return false;
+			}
+			else {
+				parSet = redis::range_as<mchlib::SetRecordDataFull>(m_redis.hscan(group_key));
+				return true;
+			}
+		}
+	}
+
+	std::vector<LocatedItem> BackendRedis::locate_in_db (const std::string& parSearch, const TagList& parTags) {
+		return dindb::locate_in_db(m_redis, parSearch, parTags);
+	}
+
+	std::vector<LocatedItem> BackendRedis::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) {
+		return dindb::locate_in_db(m_redis, parSearch, parTags);
+	}
+
+	std::vector<LocatedSet> BackendRedis::locate_sets_in_db (const std::string& parSubstr, bool parCaseInsensitive) {
+		return dindb::locate_sets_in_db(m_redis, parSubstr, parCaseInsensitive);
+	}
+
+	std::vector<LocatedSet> BackendRedis::locate_sets_in_db (const std::string& parSubstr, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
+		return dindb::locate_sets_in_db(m_redis, parSubstr, parSets, parCaseInsensitive);
+	}
+
+	std::vector<GroupIDType> BackendRedis::find_all_sets() {
+		return dindb::find_all_sets(m_redis);
+	}
+
+	std::vector<dhandy::MaxSizedArray<std::string, 4>> BackendRedis::find_set_details (const std::vector<GroupIDType>& parSets) {
+		return dindb::find_set_details(m_redis, parSets);
+	}
+
+	std::vector<dhandy::MaxSizedArray<std::string, 1>> BackendRedis::find_file_details (GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) {
+		return dindb::find_file_details(m_redis, parSetID, parLevel, parDir);
+	}
+
+	std::vector<std::string> BackendRedis::find_paths_starting_by (GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) {
+		return dindb::find_paths_starting_by(m_redis, parGroupID, parLevel, parPath);
+	}
+} //namespace dindb
+
+extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {
+	if (not parConfig)
+		return nullptr;
+
+	auto& config_node = *parConfig;
+	auto config = config_node["connection"].as<dindb::RedisConnectionSettings>();
+
+	auto vec = (config_node["script_paths"] ? config_node["script_paths"].as<std::vector<std::string>>() : std::vector<std::string>());
+	dincore::SearchPaths lua_paths(std::move(vec));
+	lua_paths.add_path(REDIS_SCRIPTS_PATH);
+
+	return new dindb::BackendRedis(
+		std::move(config.address),
+		config.port,
+		config.database,
+		true,
+		std::move(lua_paths)
+	);
+}
+
+extern "C" void dindexer_destroy_backend (dindb::Backend* parDele) {
+	if (parDele)
+		delete parDele;
+}
+
+extern "C" const char* dindexer_backend_name() {
+	return "redis";
+}
+
+extern "C" int dindexer_backend_iface_version() {
+	return dindb::g_current_iface_version;
+}

src/backends/redis/backend_redis.hpp

@@ -0,0 +1,70 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idB2F92EE07A004D5293FD0657EEE8F75B
+#define idB2F92EE07A004D5293FD0657EEE8F75B
+
+#include "backends/db_backend.hpp"
+#include "incredis/incredis.hpp"
+#include "incredis/script.hpp"
+#include "dindexer-core/searchpaths.hpp"
+#include <string>
+#include <cstdint>
+
+namespace dindb {
+	class BackendRedis : public Backend {
+	public:
+		BackendRedis ( BackendRedis&& ) = default;
+		BackendRedis ( std::string&& parAddress, uint16_t parPort, uint16_t parDatabase, bool parConnect, dincore::SearchPaths&& parLuaPaths );
+		virtual ~BackendRedis ( void ) noexcept;
+
+		virtual void connect ( void ) override;
+		virtual void disconnect ( void ) override;
+
+		virtual void tag_files ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void tag_files ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_tags ( const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_tags ( const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet ) override;
+		virtual void delete_all_tags ( const std::vector<FileIDType>& parFiles, GroupIDType parSet ) override;
+		virtual void delete_all_tags ( const std::vector<std::string>& parRegexes, GroupIDType parSet ) override;
+
+		virtual void delete_group ( const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf ) override;
+
+		virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) override;
+		virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) override;
+
+		virtual std::vector<LocatedItem> locate_in_db ( const std::string& parSearch, const TagList& parTags ) override;
+		virtual std::vector<LocatedItem> locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) override;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSubstr, bool parCaseInsensitive ) override;
+		virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSubstr, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive ) override;
+
+		virtual std::vector<GroupIDType> find_all_sets ( void ) override;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 4>> find_set_details ( const std::vector<GroupIDType>& parSets ) override;
+		virtual std::vector<dhandy::MaxSizedArray<std::string, 1>> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) override;
+		virtual std::vector<std::string> find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) override;
+
+	private:
+		redis::IncRedis m_redis;
+		redis::Script m_tag_if_in_set;
+		redis::Script m_dele_tag_if_in_set;
+		redis::Script m_dele_hash;
+		dincore::SearchPaths m_lua_script_paths;
+		uint16_t m_database;
+	};
+} //namespace dindb
+
+#endif

src/backends/redis/dele_hash.lua

@@ -0,0 +1,27 @@
+-- Copyright 2015, 2016, Michele Santullo
+-- This file is part of "dindexer".
+--
+-- "dindexer" is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- "dindexer" is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+
+local hash_key = KEYS[1]
+local base_index = ARGV[1]
+local file_count = ARGV[2]
+
+for z = base_index, base_index + file_count - 1 do
+	redis.call("SREM", hash_key, z)
+end
+if redis.call("SCARD", hash_key) == 0 then
+	redis.call("DEL", hash_key)
+end
+return nil

src/backends/redis/dele_tag_if_in_set.lua

@@ -0,0 +1,68 @@
+-- Copyright 2015, 2016, Michele Santullo
+-- This file is part of "dindexer".
+--
+-- "dindexer" is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- "dindexer" is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+
+local tag_key = KEYS[1]
+local file_key = KEYS[2]
+local group_id = ARGV[1]
+
+local function split_string(inputstr, sep)
+	if sep == nil then
+		sep = "%s"
+	end
+	local t = {}
+	local z = 1
+	for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
+		t[z] = str
+		z = z + 1
+	end
+	return t
+end
+
+local function dele_tag_from_list(tag_list, dele_tag)
+	tag_list = split_string(tag_list or "", ",")
+
+	local z = 1
+	for _,str in ipairs(tag_list) do
+		if str == dele_tag then
+			table.remove(tag_list, z)
+			break
+		end
+		z = z + 1
+	end
+	return table.concat(tag_list, ",")
+end
+
+if group_id ~= 0 then
+	local found_group_id = redis.call("HGET", file_key, "group_id")
+	if found_group_id ~= group_id then
+		return nil
+	end
+end
+
+local dele_tag = split_string(tag_key, ":")
+local new_tag_list = dele_tag_from_list(redis.call("HGET", file_key, "tags"), dele_tag[#dele_tag])
+
+if new_tag_list == "" then
+	redis.call("HDEL", file_key, "tags")
+else
+	redis.call("HSET", file_key, "tags", new_tag_list)
+end
+
+local retval = redis.call("SREM", tag_key, file_key)
+if redis.call("SCARD", tag_key) == 0 then
+	redis.call("DEL", tag_key)
+end
+return retval

src/backends/redis/delete.cpp

@@ -0,0 +1,153 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "delete.hpp"
+#include "tag.hpp"
+#include "incredis/incredis.hpp"
+#include "duckhandy/lexical_cast.hpp"
+#include "duckhandy/sequence_bt.hpp"
+#include "dindexerConfig.h"
+#include <vector>
+#include <tuple>
+#include <utility>
+#include <boost/iterator/counting_iterator.hpp>
+#include <algorithm>
+
+namespace dindb {
+	namespace {
+		std::pair<bool, std::size_t> confirm_dele (redis::IncRedisBatch& parBatch, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+			using dhandy::lexical_cast;
+
+			if (parIDs.empty())
+				return std::make_pair(false, parIDs.size());
+
+			for (auto id : parIDs) {
+				const auto set_key = PROGRAM_NAME ":set:" + lexical_cast<std::string>(id);
+				parBatch.hmget(set_key, "base_file_id", "item_count", "name");
+			}
+
+			std::map<GroupIDType, std::string> set_dele_list;
+			std::size_t id_index = 0;
+			for (const auto& reply : parBatch.replies()) {
+				const auto res = redis::get_array(reply);
+				assert(res.size() == 3);
+				if (redis::RedisVariantType_Nil != res[0].which()) {
+					assert(redis::RedisVariantType_Nil != res[1].which());
+					set_dele_list[parIDs[id_index]] = redis::get_string(res[2]);
+				}
+				++id_index;
+			}
+
+			if (set_dele_list.empty())
+				return std::make_pair(false, set_dele_list.size());
+			else
+				return std::make_pair(parConf(set_dele_list), set_dele_list.size());
+				//return std::make_pair(true, set_dele_list.size());
+		}
+
+		template <typename IT, IT CHUNK, typename F, IT... SVALS>
+		void chunked_run_pvt (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey, dhandy::bt::number_seq<IT, SVALS...>) {
+			for (IT i = 0; i < parCount / CHUNK; ++i) {
+				parBatch.run(parCommand, parMakeKey(i * CHUNK + parFrom + SVALS)...);
+			}
+			for (auto i = ((parFrom + parCount) / CHUNK) * CHUNK; i < parFrom + parCount; ++i) {
+				const auto key = parMakeKey(i);
+				parBatch.run(parCommand, key);
+			}
+		}
+
+		template <typename IT, IT CHUNK, typename F>
+		void chunked_run (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey) {
+			chunked_run_pvt<IT, CHUNK, F>(parBatch, parCommand, parFrom, parCount, parMakeKey, dhandy::bt::number_range<IT, 0, CHUNK>());
+		};
+	} //unnamed namespace
+
+	void delete_group_from_db (redis::IncRedis& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+		using dhandy::lexical_cast;
+		using IDRange = std::tuple<GroupIDType, FileIDType, FileIDType>;
+
+		auto set_batch = parRedis.make_batch();
+
+		auto dele_pair = confirm_dele(set_batch, parIDs, parConf);
+		assert(set_batch.batch().replies_requested());
+		if (not dele_pair.first)
+			return;
+
+		std::vector<IDRange> ranges;
+		ranges.reserve(dele_pair.second);
+		std::size_t id_index = 0;
+		for (const auto& reply : set_batch.replies()) {
+			const auto res = redis::get_array(reply);
+			if (redis::RedisVariantType_Nil != res[0].which()) {
+				ranges.push_back(
+					IDRange(
+						parIDs[id_index],
+						lexical_cast<FileIDType>(redis::get_string(res[0])),
+						lexical_cast<FileIDType>(redis::get_string(res[1]))
+					)
+				);
+			}
+			++id_index;
+		}
+		assert(ranges.size() == dele_pair.second);
+
+		for (const auto& dele_tuple : ranges) {
+			const auto set_id = std::get<0>(dele_tuple);
+			const auto file_base_index = std::get<1>(dele_tuple);
+			const auto file_count = std::get<2>(dele_tuple);
+
+			std::vector<FileIDType> ids;
+			ids.reserve(file_count);
+			std::copy(
+				boost::counting_iterator<FileIDType>(file_base_index),
+				boost::counting_iterator<FileIDType>(file_base_index + file_count),
+				std::back_inserter(ids)
+			);
+
+			delete_all_tags(parRedis, parDeleTagIfInSet, ids, set_id);
+		}
+
+		auto dele_batch = parRedis.make_batch();
+		for (const auto& dele_tuple : ranges) {
+			const auto set_id = std::get<0>(dele_tuple);
+			const auto file_base_index = std::get<1>(dele_tuple);
+			const auto file_count = std::get<2>(dele_tuple);
+
+			auto hash_query_batch = parRedis.make_batch();
+			for (FileIDType i = file_base_index; i < file_base_index + file_count; ++i) {
+				const auto file_key = PROGRAM_NAME ":file:" + lexical_cast<std::string>(i);
+				hash_query_batch.hget(file_key, "hash");
+			}
+			hash_query_batch.throw_if_failed();
+
+			for (const auto& rep : hash_query_batch.replies()) {
+				const auto hash_key = PROGRAM_NAME ":hash:" + redis::get_string(rep);
+				parDeleHash.run(
+					dele_batch.batch(),
+					std::make_tuple(hash_key),
+					std::make_tuple(lexical_cast<std::string>(file_base_index), lexical_cast<std::string>(file_count))
+				);
+			}
+
+			dele_batch.del(PROGRAM_NAME ":set:" + lexical_cast<std::string>(set_id));
+			dele_batch.del(PROGRAM_NAME ":levels:" + lexical_cast<std::string>(set_id));
+			chunked_run<FileIDType, 8>(dele_batch.batch(), +"DEL", file_base_index, file_count, [](FileIDType id){return PROGRAM_NAME ":file:" + lexical_cast<std::string>(id);});
+		}
+
+		dele_batch.throw_if_failed();
+	}
+} //namespace dindb

src/backends/redis/delete.hpp

@@ -0,0 +1,45 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id1E0F8FEB88EC4FED843FFEE7BAB624BB
+#define id1E0F8FEB88EC4FED843FFEE7BAB624BB
+
+#include "backends/db_backend.hpp"
+#include <functional>
+#include <vector>
+#include <string>
+#include <map>
+
+namespace redis {
+	class IncRedis;
+	class Script;
+} //namespace redis
+
+namespace dindb {
+	using IDDescMap = std::map<GroupIDType, std::string>;
+	using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
+
+	void delete_group_from_db (
+		redis::IncRedis& parRedis,
+		redis::Script& parDeleTagIfInSet,
+		redis::Script& parDeleHash,
+		const std::vector<GroupIDType>& parIDs,
+		ConfirmDeleCallback parConf
+	);
+} //namespace dindb
+
+#endif

src/backends/redis/find.cpp

@@ -0,0 +1,331 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "find.hpp"
+#include "incredis/incredis.hpp"
+#include "duckhandy/lexical_cast.hpp"
+#include "dindexerConfig.h"
+#include "dindexer-core/split_tags.hpp"
+#include "dindexer-machinery/tiger.hpp"
+#include "duckhandy/compatibility.h"
+#include <boost/regex.hpp>
+#include <ciso646>
+#include <algorithm>
+#include <boost/range/adaptor/filtered.hpp>
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/iterator/zip_iterator.hpp>
+#include <functional>
+#include <iterator>
+#include <cstdint>
+#include <boost/algorithm/string/predicate.hpp>
+
+namespace dindb {
+	namespace {
+		inline std::string to_std_string ( boost::string_ref parStr ) a_always_inline;
+		template <typename T>
+		inline void concatenate ( std::vector<T>&& parAppend, std::vector<T>& parOut ) a_always_inline;
+		template <typename T>
+		inline T construct ( const std::vector<redis::Reply>& parData, const std::string& parID ) a_always_inline;
+		template <>
+		inline LocatedItem construct ( const std::vector<redis::Reply>& parData, const std::string& parID ) a_always_inline;
+		template <>
+		inline LocatedSet construct ( const std::vector<redis::Reply>& parData, const std::string& parID ) a_always_inline;
+
+		bool all_tags_match (const TagList& parTags, const std::string& parTaglist) {
+			const auto tags = dincore::split_tags(parTaglist);
+
+			if (tags.size() >= parTags.size()) {
+				for (const auto& required_tag : parTags) {
+					if (std::find(tags.begin(), tags.end(), required_tag) == tags.end()) {
+						return false;
+					}
+				}
+			}
+			else {
+				return false;
+			}
+			return true;
+		}
+
+		//See: http://stackoverflow.com/questions/12552277/whats-the-best-way-to-iterate-over-two-or-more-containers-simultaneously/12553437#12553437
+		//(referenced from http://stackoverflow.com/questions/16982190/c-use-boost-range-transformed-adaptor-with-binary-function)
+		//What became of this? http://marc.info/?l=boost-users&m=129619765731342
+		template <typename... Conts>
+		auto zip_range (Conts&... parConts) -> decltype(boost::make_iterator_range(
+			boost::make_zip_iterator(boost::make_tuple(parConts.begin()...)),
+		    boost::make_zip_iterator(boost::make_tuple(parConts.end()...)))
+		) {
+			return {
+				boost::make_zip_iterator(boost::make_tuple(parConts.begin()...)),
+				boost::make_zip_iterator(boost::make_tuple(parConts.end()...))
+			};
+		}
+
+		template <typename T>
+		std::vector<T> store_filtered_items (
+			const std::vector<redis::Reply>& parReplies,
+			const std::vector<std::string>& parIDs,
+			std::function<bool(const boost::tuple<std::vector<redis::Reply>, std::string>&)> parFilter
+		) {
+			using boost::adaptors::filtered;
+			using boost::adaptors::transformed;
+			using boost::tuple;
+			using boost::make_tuple;
+			using redis::Reply;
+			using std::vector;
+			using dhandy::lexical_cast;
+
+			assert(parReplies.size() == parIDs.size());
+			return boost::copy_range<vector<T>>(
+				zip_range(parReplies, parIDs) |
+				transformed([](const tuple<Reply, std::string>& r) {
+					return make_tuple(redis::get_array(r.get<0>()), r.get<1>());
+				}) |
+				filtered(parFilter) |
+				transformed([](const tuple<vector<Reply>, std::string>& t) {
+					return construct<T>(t.get<0>(), t.get<1>());
+				})
+			);
+		}
+
+		std::string to_std_string (boost::string_ref parStr) {
+			return std::string(parStr.data(), parStr.size());
+		}
+
+		template <typename T>
+		void concatenate (std::vector<T>&& parAppend, std::vector<T>& parOut) {
+			parOut.insert(parOut.end(), std::make_move_iterator(parAppend.begin()), std::make_move_iterator(parAppend.end()));
+		}
+
+		template <>
+		LocatedItem construct (const std::vector<redis::Reply>& parData, const std::string& parID) {
+			using dhandy::lexical_cast;
+
+			//parData is expected to contain: "path", "group_id", "tags"
+
+			assert(parData.size() == 3);
+			return LocatedItem{
+				redis::get_string(parData[0]),
+				lexical_cast<FileIDType>(parID),
+				lexical_cast<GroupIDType>(redis::get_string(parData[1]))
+			};
+		}
+
+		template <>
+		LocatedSet construct (const std::vector<redis::Reply>& parData, const std::string& parID) {
+			using dhandy::lexical_cast;
+
+			//parData is expected to contain: "desc", "item_count", "dir_count"
+
+			assert(parData.size() == 3);
+			const auto itm_count = lexical_cast<uint32_t>(redis::get_string(parData[1]));
+			const auto dir_count = lexical_cast<uint32_t>(redis::get_string(parData[2]));
+			assert(dir_count <= itm_count);
+
+			return LocatedSet{
+				redis::get_string(parData[0]),
+				lexical_cast<GroupIDType>(parID),
+				itm_count - dir_count,
+				dir_count
+			};
+		}
+
+		template <typename T, typename F, typename... FIELDS>
+		std::vector<T> locate_in_bursts (redis::IncRedis& parRedis, const char* parScanKeyFilter, F parFilter, FIELDS&&... parFields) {
+			using dincore::split_and_trim;
+
+			const int prefetch_count = 500;
+			std::vector<T> retval;
+			std::vector<std::string> ids;
+			ids.reserve(prefetch_count);
+
+			int curr_count = 0;
+			auto batch = parRedis.make_batch();
+			for (const auto& itm : parRedis.scan(parScanKeyFilter)) {
+				++curr_count;
+				batch.hmget(itm, std::forward<FIELDS>(parFields)...);
+				ids.push_back(to_std_string(split_and_trim(itm, ':').back()));
+
+				if (curr_count == prefetch_count) {
+					concatenate(store_filtered_items<T>(batch.replies(), ids, parFilter), retval);
+					batch.reset();
+					curr_count = 0;
+					ids.clear();
+				}
+			}
+			if (curr_count)
+				concatenate(store_filtered_items<T>(batch.replies(), ids, parFilter), retval);
+			return retval;
+		}
+	} //unnamed namespace
+
+	std::vector<GroupIDType> find_all_sets (redis::IncRedis& parRedis) {
+		using dincore::split_and_trim;
+
+		std::vector<GroupIDType> retval;
+		for (const auto& itm : parRedis.scan(PROGRAM_NAME ":set:*")) {
+			retval.push_back(dhandy::lexical_cast<GroupIDType>(split_and_trim(itm, ':').back()));
+		}
+		return retval;
+	}
+
+	std::vector<LocatedItem> locate_in_db (redis::IncRedis& parRedis, const std::string& parRegex, const TagList& parTags) {
+		const boost::regex search(parRegex, boost::regex_constants::optimize | boost::regex_constants::nosubs | boost::regex_constants::perl);
+
+		auto filter = [&parTags, &search](const boost::tuple<std::vector<redis::Reply>, std::string>& t) {
+			return (parTags.empty() or all_tags_match(parTags, redis::get_string(t.get<0>()[2]))) and
+				boost::regex_search(redis::get_string(t.get<0>()[0]), search);
+		};
+
+		return locate_in_bursts<LocatedItem>(parRedis, PROGRAM_NAME ":file:*", filter, "path", "group_id", "tags");
+	}
+
+	std::vector<LocatedItem> locate_in_db (redis::IncRedis& parRedis, const mchlib::TigerHash& parSearch, const TagList& parTags) {
+		const auto hash_key = PROGRAM_NAME ":hash:" + mchlib::tiger_to_string(parSearch, false);
+		const auto file_ids = parRedis.smembers(hash_key);
+
+		std::vector<std::string> ids;
+		if (file_ids) {
+			auto batch = parRedis.make_batch();
+			for (auto&& file_id : *file_ids) {
+				if (not file_id)
+					continue;
+
+				const auto file_key = PROGRAM_NAME ":file:" + *file_id;
+				ids.emplace_back(std::move(*file_id));
+				batch.hmget(file_key, "path", "group_id", "tags");
+			}
+			batch.throw_if_failed();
+
+			return store_filtered_items<LocatedItem>(
+				batch.replies(),
+				ids,
+				[&parTags](const boost::tuple<std::vector<redis::Reply>, std::string>& t) {
+					return parTags.empty() or all_tags_match(parTags, redis::get_string(t.get<0>()[2]));
+				}
+			);
+		}
+		else {
+			return std::vector<LocatedItem>();
+		}
+	}
+
+	std::vector<LocatedSet> locate_sets_in_db (redis::IncRedis& parRedis, const std::string& parSubstr, bool parCaseInsensitive) {
+		return locate_sets_in_db(parRedis, parSubstr, std::vector<GroupIDType>(), parCaseInsensitive);
+	}
+
+	std::vector<LocatedSet> locate_sets_in_db (redis::IncRedis& parRedis, const std::string& parSubstr, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
+		using dhandy::lexical_cast;
+
+		auto filter_case_ins = [&parSubstr, &parSets](const boost::tuple<std::vector<redis::Reply>, std::string>& t) {
+			const auto& s = redis::get_string(t.get<0>()[0]);
+			return s.end() != std::search(
+				s.begin(),
+				s.end(),
+				parSubstr.begin(),
+				parSubstr.end(),
+				[](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);}) and
+				(
+					parSets.empty() or
+					std::find(parSets.begin(), parSets.end(), lexical_cast<GroupIDType>(t.get<1>())) != parSets.end()
+				);
+		};
+		auto filter_case_sens = [&parSubstr, &parSets](const boost::tuple<std::vector<redis::Reply>, std::string>& t) {
+			return redis::get_string(t.get<0>()[0]).find(parSubstr) != std::string::npos and
+				(
+					parSets.empty() or
+					std::find(parSets.begin(), parSets.end(), lexical_cast<GroupIDType>(t.get<1>())) != parSets.end()
+				);
+		};
+		std::function<bool(const boost::tuple<std::vector<redis::Reply>, std::string>&)> filter;
+		if (parCaseInsensitive)
+			filter = filter_case_ins;
+		else
+			filter = filter_case_sens;
+
+		return locate_in_bursts<LocatedSet>(parRedis, PROGRAM_NAME ":set:*", filter, "desc", "item_count", "dir_count");
+	}
+
+	std::vector<dhandy::MaxSizedArray<std::string, 4>> find_set_details (redis::IncRedis& parRedis, const std::vector<GroupIDType>& parSets) {
+		using dhandy::lexical_cast;
+
+		auto batch = parRedis.make_batch();
+		for (auto set_id : parSets) {
+			const auto set_key = PROGRAM_NAME ":set:" + lexical_cast<std::string>(set_id);
+			batch.hmget(set_key, "creation", "name", "disk_label");
+		}
+		batch.throw_if_failed();
+
+		std::vector<dhandy::MaxSizedArray<std::string, 4>> retval;
+		auto curr_set = parSets.begin();
+		for (const auto& reply : batch.replies()) {
+			const auto& reply_list = get_array(reply);
+			if (redis::RedisVariantType_Nil != reply_list[0].which() and
+				redis::RedisVariantType_Nil != reply_list[1].which() and
+				redis::RedisVariantType_Nil != reply_list[2].which())
+			{
+				retval.resize(retval.size() + 1);
+				retval.back().push_back(lexical_cast<std::string>(*curr_set));
+				retval.back().push_back(get_string(reply_list[1]));
+				retval.back().push_back(get_string(reply_list[0]));
+				retval.back().push_back(get_string(reply_list[2]));
+			}
+			++curr_set;
+		}
+		return retval;
+	};
+
+	std::vector<dhandy::MaxSizedArray<std::string, 1>> find_file_details (redis::IncRedis& parRedis, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) {
+		using dhandy::lexical_cast;
+		using RetListType = std::vector<dhandy::MaxSizedArray<std::string, 1>>;
+
+		const double level = static_cast<float>(parLevel);
+		auto lst = parRedis.zrangebyscore(PROGRAM_NAME ":levels:" + lexical_cast<std::string>(parSetID), level, true, level, true, false);
+		if (not lst)
+			return RetListType();
+
+		auto batch = parRedis.make_batch();
+		for (const auto& itm : *lst) {
+			if (itm)
+				batch.hget(*itm, "path");
+		}
+		batch.throw_if_failed();
+
+		std::vector<dhandy::MaxSizedArray<std::string, 1>> retval;
+		for (auto& reply : batch.replies()) {
+			if (redis::RedisVariantType_Nil != reply.which()) {
+				auto curr_path = get_string(reply);
+				if (boost::starts_with(curr_path, parDir)) {
+					retval.resize(retval.size() + 1);
+					retval.back().push_back(std::move(curr_path));
+				}
+			}
+		}
+		return retval;
+	};
+
+	std::vector<std::string> find_paths_starting_by (redis::IncRedis& parRedis, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) {
+		using boost::adaptors::transformed;
+		using dhandy::MaxSizedArray;
+
+		auto file_details = find_file_details(parRedis, parGroupID, parLevel, parPath);
+		return boost::copy_range<std::vector<std::string>>(
+			file_details |
+			transformed([](MaxSizedArray<std::string, 1>& a){return std::move(a.front());})
+		);
+	}
+} //namespace dindb

src/backends/redis/find.hpp

@@ -0,0 +1,43 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idB4972996B4494E66A03B6AE205B1FA36
+#define idB4972996B4494E66A03B6AE205B1FA36
+
+#include "backends/db_backend.hpp"
+#include "duckhandy/MaxSizedArray.hpp"
+#include <vector>
+#include <string>
+#include <boost/utility/string_ref.hpp>
+#include <cstdint>
+
+namespace redis {
+	class IncRedis;
+} //namespace redis
+
+namespace dindb {
+	std::vector<GroupIDType> find_all_sets ( redis::IncRedis& parRedis );
+	std::vector<LocatedItem> locate_in_db ( redis::IncRedis& parRedis, const std::string& parRegex, const TagList& parTags );
+	std::vector<LocatedItem> locate_in_db ( redis::IncRedis& parRedis, const mchlib::TigerHash& parSearch, const TagList& parTags );
+	std::vector<LocatedSet> locate_sets_in_db ( redis::IncRedis& parRedis, const std::string& parSubstr, bool parCaseInsensitive );
+	std::vector<LocatedSet> locate_sets_in_db ( redis::IncRedis& parRedis, const std::string& parSubstr, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive );
+	std::vector<dhandy::MaxSizedArray<std::string, 4>> find_set_details ( redis::IncRedis& parRedis, const std::vector<GroupIDType>& parSets );
+	std::vector<dhandy::MaxSizedArray<std::string, 1>> find_file_details ( redis::IncRedis& parRedis, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir );
+	std::vector<std::string> find_paths_starting_by ( redis::IncRedis& parRedis, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath );
+} //namespace dindb
+
+#endif

src/backends/redis/record_data_adapt.hpp

@@ -0,0 +1,97 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id2949D72CC2F246D4A289FFB820CC3A8F
+#define id2949D72CC2F246D4A289FFB820CC3A8F
+
+#include "dindexer-machinery/recorddata.hpp"
+#include "duckhandy/lexical_cast.hpp"
+#include <array>
+#include <string>
+#include <cstdint>
+#include <utility>
+
+namespace dindb {
+	using FileRecordDataWithGroup = std::pair<std::string, mchlib::FileRecordData>;
+} //namespace dindb
+
+namespace redis {
+	template <>
+	struct StructAdapt<dindb::FileRecordDataWithGroup> {
+		template <typename R>
+		static bool decode (const R& parRange, dindb::FileRecordDataWithGroup& parOut) {
+			using dhandy::lexical_cast;
+
+			std::array<std::string, 2> mime;
+			std::string group_key;
+
+			for (const auto itm : parRange) {
+				if (itm.first == "path")
+					parOut.second.abs_path = itm.second;
+				else if (itm.first == "hash")
+					parOut.second.hash = mchlib::string_to_tiger(itm.second);
+				else if (itm.first == "size")
+					parOut.second.size = lexical_cast<decltype(parOut.second.size)>(itm.second);
+				else if (itm.first == "level")
+					parOut.second.level = lexical_cast<decltype(parOut.second.level)>(itm.second);
+				else if (itm.first == "mime_type")
+					mime[0] = itm.second;
+				else if (itm.first == "mime_charset")
+					mime[1] = itm.second;
+				else if (itm.first == "is_directory")
+					parOut.second.is_directory = (itm.second[0] == '0' ? false : true);
+				else if (itm.first == "is_symlink")
+					parOut.second.is_symlink = (itm.second[0] == '0' ? false : true);
+				else if (itm.first == "unreadable")
+					parOut.second.unreadable = (itm.second[0] == '0' ? false : true);
+				else if (itm.first == "hash_valid")
+					parOut.second.hash_valid = (itm.second[0] == '0' ? false : true);
+				else if (itm.first == "group_id")
+					parOut.first = itm.second;
+			}
+			parOut.second.mime_full = mime[0] + mime[1];
+			parOut.second.mime_type_offset = 0;
+			parOut.second.mime_type_length = parOut.second.mime_charset_offset = static_cast<uint16_t>(mime[0].size());
+			parOut.second.mime_charset_length = static_cast<uint16_t>(mime[1].size());
+			return true;
+		}
+	};
+
+	template <>
+	struct StructAdapt<mchlib::SetRecordDataFull> {
+		template <typename R>
+		static bool decode (const R& parRange, mchlib::SetRecordDataFull& parOut) {
+			using dhandy::lexical_cast;
+
+			for (const auto& itm : parRange) {
+				if (itm.first == "name")
+					parOut.name = itm.second;
+				else if (itm.first == "disk_label")
+					parOut.disk_label = itm.second;
+				else if (itm.first == "fs_uuid")
+					parOut.fs_uuid = itm.second;
+				else if (itm.first == "type")
+					parOut.type = itm.second[0];
+				else if (itm.first == "content_type")
+					parOut.content_type = itm.second[0];
+			}
+			return true;
+		}
+	};
+} //namespace redis
+
+#endif

src/backends/redis/redisConfig.h.in

@@ -15,10 +15,9 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef id8B9FF1807C05459DAE8CD05312147416
-#define id8B9FF1807C05459DAE8CD05312147416
+#ifndef idB3E2F339E3B64378B66342550C4D2089
+#define idB3E2F339E3B64378B66342550C4D2089
 
-#define STRINGIZE_IMPL(s) #s
-#define STRINGIZE(s) STRINGIZE_IMPL(s)
+#define REDIS_SCRIPTS_PATH "@DINDEXER_REDIS_SCRIPTS_PATH@"
 
 #endif

src/backends/redis/tag.cpp

@@ -0,0 +1,193 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tag.hpp"
+#include "incredis/incredis.hpp"
+#include "dindexerConfig.h"
+#include "duckhandy/lexical_cast.hpp"
+#include "dindexer-core/split_tags.hpp"
+#include <sstream>
+#include <tuple>
+#include <boost/regex.hpp>
+#include <unordered_set>
+#include <boost/functional/hash.hpp>
+
+namespace std {
+    template<>
+    struct hash<boost::string_ref> {
+        size_t operator() (boost::string_ref const& sr) const {
+            return boost::hash_range(sr.begin(), sr.end());
+        }
+    };
+} //namespace std
+
+namespace dindb {
+	namespace {
+		std::string make_file_key (FileIDType parID) {
+			return PROGRAM_NAME ":file:" + dhandy::lexical_cast<std::string>(parID);
+		}
+
+		std::vector<boost::regex> compile_regexes (const std::vector<std::string>& parRegexes) {
+			std::vector<boost::regex> retval;
+			retval.reserve(parRegexes.size());
+			for (const auto& str : parRegexes) {
+				retval.emplace_back(boost::regex(
+					str,
+					boost::regex_constants::optimize | boost::regex_constants::nosubs | boost::regex_constants::perl
+				));
+			}
+			assert(retval.size() == parRegexes.size());
+			return retval;
+		}
+
+		void run_id_based_script (redis::IncRedis& parRedis, redis::Script& parScript, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+			using dhandy::lexical_cast;
+
+			auto batch = parRedis.command().make_batch();
+			const std::string set_id = lexical_cast<std::string>(parSet);
+			for (const auto file_id : parFiles) {
+				for (const auto &tag : parTags) {
+					std::ostringstream oss;
+					oss << PROGRAM_NAME ":tag:" << tag;
+					const std::string tag_key = oss.str();
+					const std::string file_key = make_file_key(file_id);
+					parScript.run(batch, std::make_tuple(tag_key, file_key), std::make_tuple(set_id));
+				}
+			}
+
+			batch.throw_if_failed();
+		}
+
+		void run_regex_based_script(redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+			using dhandy::lexical_cast;
+
+			const std::string set_id = lexical_cast<std::string>(parSet);
+			const auto regexes = compile_regexes(parRegexes);
+			for (const auto &itm : parRedis.scan(PROGRAM_NAME ":file:*")) {
+				const auto &file_key = itm;
+				const auto path = parRedis.hget(file_key, "path");
+
+				auto batch = parRedis.command().make_batch();
+				for (const auto &regex : regexes) {
+					if (not path or not boost::regex_search(*path, regex))
+						continue;
+
+					for (const auto &tag : parTags) {
+						std::ostringstream oss;
+						oss << PROGRAM_NAME ":tag:" << tag;
+						const std::string tag_key = oss.str();
+						parTagIfInSet.run(batch, std::make_tuple(tag_key, file_key), std::make_tuple(set_id));
+					}
+					break;
+				}
+				batch.throw_if_failed();
+			}
+		}
+
+		template <typename T>
+		T id_from_redis_key (const std::string& parKey) {
+			assert(not parKey.empty());
+			return dhandy::lexical_cast<T>(dincore::split_and_trim(parKey, ':').back());
+		}
+	} //unnamed namespace
+
+	void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		run_id_based_script(parRedis, parTagIfInSet, parFiles, parTags, parSet);
+	}
+
+	void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		run_regex_based_script(parRedis, parTagIfInSet, parRegexes, parTags, parSet);
+	}
+
+	void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		run_id_based_script(parRedis, parDeleIfInSet, parFiles, parTags, parSet);
+	}
+
+	void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
+		run_regex_based_script(parRedis, parDeleIfInSet, parRegexes, parTags, parSet);
+	}
+
+	void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
+		auto batch = parRedis.make_batch();
+		for (const auto file_id : parFiles) {
+			const auto file_key = make_file_key(file_id);
+			batch.hget(file_key, "tags");
+		}
+
+		batch.throw_if_failed();
+		std::unordered_set<boost::string_ref> dele_tags;
+		for (const auto& reply : batch.replies()) {
+			auto tags = dincore::split_tags(redis::get_string(reply));
+			for (const auto& tag : tags) {
+				dele_tags.insert(tag);
+			}
+		}
+
+		std::vector<boost::string_ref> vec_dele_tags(dele_tags.begin(), dele_tags.end());
+		delete_tags(parRedis, parDeleIfInSet, parFiles, vec_dele_tags, parSet);
+	}
+
+	void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, GroupIDType parSet) {
+		using dhandy::lexical_cast;
+
+		const auto regexes = compile_regexes(parRegexes);
+
+		std::unordered_set<std::string> dele_tags;
+		std::vector<FileIDType> ids;
+
+		for (const auto& itm : parRedis.scan(PROGRAM_NAME ":file:*")) {
+			const auto& file_key = itm;
+			auto opt_file_replies = parRedis.hmget(file_key, "path", "tags", "group_id");
+			assert(opt_file_replies and opt_file_replies->size() == 3);
+			if (not opt_file_replies)
+				continue;
+			const auto& file_replies = *opt_file_replies;
+			const auto group_id = lexical_cast<GroupIDType>(*file_replies[2]);
+			if (parSet != InvalidGroupID and parSet != group_id)
+				continue;
+
+			const auto& path = *file_replies[0];
+			const auto tags_str = (file_replies[1] ? std::string() : *file_replies[1]);
+			const auto tags = dincore::split_tags(tags_str);
+			const auto file_id = id_from_redis_key<FileIDType>(file_key);
+
+			for (const auto &regex : regexes) {
+				if (not boost::regex_search(path, regex))
+					continue;
+
+				ids.push_back(file_id);
+
+				for (const auto &tag : tags) {
+					dele_tags.insert(std::string(tag.data(), tag.size()));
+				}
+				break;
+			}
+		}
+
+		std::vector<boost::string_ref> dele_tags_vec;
+		dele_tags_vec.reserve(dele_tags.size());
+		std::transform(
+			dele_tags.begin(),
+			dele_tags.end(),
+			std::back_inserter(dele_tags_vec),
+			[](const std::string& parS) { return boost::string_ref(parS); }
+		);
+
+		delete_tags(parRedis, parDeleIfInSet, ids, dele_tags_vec, parSet);
+	}
+} //namespace dindb
+

src/backends/redis/tag.hpp

@@ -0,0 +1,74 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id2874EA620CF0415CAF5B005E227BC44B
+#define id2874EA620CF0415CAF5B005E227BC44B
+
+#include "backends/db_backend.hpp"
+#include <vector>
+#include <boost/utility/string_ref.hpp>
+
+namespace redis {
+	class IncRedis;
+	class Script;
+} //namespace redis
+
+namespace dindb {
+	void tag_files (
+		redis::IncRedis& parRedis,
+		redis::Script& parTagIfInSet,
+		const std::vector<FileIDType>& parFiles,
+		const std::vector<boost::string_ref>& parTags,
+		GroupIDType parSet
+	);
+	void tag_files (
+		redis::IncRedis& parRedis,
+		redis::Script& parTagIfInSet,
+		const std::vector<std::string>& parRegexes,
+		const std::vector<boost::string_ref>& parTags,
+		GroupIDType parSet
+	);
+
+	void delete_tags (
+		redis::IncRedis& parRedis,
+		redis::Script& parDeleIfInSet,
+		const std::vector<FileIDType>& parFiles,
+		const std::vector<boost::string_ref>& parTags,
+		GroupIDType parSet
+	);
+	void delete_tags (
+		redis::IncRedis& parRedis,
+		redis::Script& parDeleIfInSet,
+		const std::vector<std::string>& parRegexes,
+		const std::vector<boost::string_ref>& parTags,
+		GroupIDType parSet
+	);
+	void delete_all_tags (
+		redis::IncRedis& parRedis,
+		redis::Script& parDeleIfInSet,
+		const std::vector<FileIDType>& parFiles,
+		GroupIDType parSet
+	);
+	void delete_all_tags (
+		redis::IncRedis& parRedis,
+		redis::Script& parDeleIfInSet,
+		const std::vector<std::string>& parRegexes,
+		GroupIDType parSet
+	);
+} //namespace dindb
+
+#endif

src/backends/redis/tag_if_in_set.lua

@@ -0,0 +1,65 @@
+-- Copyright 2015, 2016, Michele Santullo
+-- This file is part of "dindexer".
+--
+-- "dindexer" is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- "dindexer" is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+
+local tag_key = KEYS[1]
+local file_key = KEYS[2]
+local group_id = ARGV[1]
+
+local function split_string(inputstr, sep)
+	if sep == nil then
+		sep = "%s"
+	end
+	local t = {}
+	local z = 1
+	for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
+		t[z] = str
+		z = z + 1
+	end
+	return t
+end
+
+local function add_tag_to_list(tag_list, new_tag)
+	if tag_list == false or tag_list == "" then
+		return new_tag
+	end
+	tag_list = split_string(tag_list, ",")
+
+	local new_tag_found = false
+	for _,str in ipairs(tag_list) do
+		if str == new_tag then
+			new_tag_found = true
+			break
+		end
+	end
+	if not new_tag_found then
+		tag_list[#tag_list+1] = new_tag
+	end
+	table.sort(tag_list)
+	return table.concat(tag_list, ",")
+end
+
+if group_id ~= 0 then
+	local found_group_id = redis.call("HGET", file_key, "group_id")
+	if found_group_id ~= group_id then
+		return nil
+	end
+end
+
+local new_tag = split_string(tag_key, ":")
+local new_tag_list = add_tag_to_list(redis.call("HGET", file_key, "tags"), new_tag[#new_tag])
+
+redis.call("HSET", file_key, "tags", new_tag_list)
+return redis.call("SADD", tag_key, file_key)

src/common/CMakeLists.txt

@@ -1,4 +1,4 @@
-project(${bare_name}-common CXX C)
+project(${bare_name}-common CXX)
 
 find_package(Readline 6.3 REQUIRED)
 
@@ -8,7 +8,6 @@ add_library(${PROJECT_NAME}
 	validationerror.cpp
 	common_info.cpp
 	readline_wrapper.cpp
-	split_tags.cpp
 )
 
 target_include_directories(${PROJECT_NAME}
@@ -23,6 +22,8 @@ target_link_libraries(${PROJECT_NAME}
 	PRIVATE ${bare_name}-if
 	PRIVATE ${YAMLCPP_LIBRARY}
 	PRIVATE ${Readline_LIBRARY}
+	${bare_name}-backend
+	${bare_name}-core
 )
 
 #install(TARGETS ${PROJECT_NAME}
@@ -31,7 +32,7 @@ target_link_libraries(${PROJECT_NAME}
 #	ARCHIVE DESTINATION lib/static
 #)
 
-#Allow to link with .so
+#Allow linking with .so
 #see https://cmake.org/pipermail/cmake/2007-May/014350.html
 #and http://stackoverflow.com/questions/6093547/what-do-r-x86-64-32s-and-r-x86-64-64-relocation-mean/6093910#6093910
 if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" )

src/common/commandline.cpp

@@ -18,9 +18,9 @@
 #include "dindexer-common/commandline.hpp"
 #include "dindexerConfig.h"
 #include "gitinfo.h"
-#include "helpers/lengthof.h"
-#include "helpers/stringize.h"
-#include "helpers/infix_iterator.hpp"
+#include "duckhandy/lengthof.h"
+#include "duckhandy/stringize.h"
+#include "duckhandy/infix_iterator.hpp"
 #include <boost/program_options.hpp>
 #include <ostream>
 #include <utility>

src/common/common_info.cpp

@@ -17,7 +17,7 @@
 
 #include "dindexer-common/common_info.hpp"
 #include "dindexerConfig.h"
-#include "helpers/stringize.h"
+#include "duckhandy/stringize.h"
 
 namespace dinlib {
 	namespace {

src/common/settings.cpp

@@ -16,59 +16,46 @@
  */
 
 #include "dindexer-common/settings.hpp"
+#include "dindexer-core/split_tags.hpp"
+#include "dindexer-core/searchpaths.hpp"
+#include "dindexerConfig.h"
 #include <yaml-cpp/yaml.h>
 #include <ciso646>
 #include <wordexp.h>
-
-namespace YAML {
-	template<>
-	struct convert<dinlib::SettingsDB> {
-		static Node encode (const dinlib::SettingsDB& parSettings) {
-			Node node;
-			node["address"] = parSettings.address;
-			node["username"] = parSettings.username;
-			node["password"] = parSettings.password;
-			node["port"] = parSettings.port;
-			node["dbname"] = parSettings.dbname;
-			return node;
-		}
-
-		static bool decode (const Node& parNode, dinlib::SettingsDB& parSettings) {
-			if (not parNode.IsMap() or parNode.size() != 5) {
-				return false;
-			}
-
-			parSettings.address = parNode["address"].as<std::string>();
-			parSettings.username = parNode["username"].as<std::string>();
-			parSettings.password = parNode["password"].as<std::string>();
-			parSettings.dbname = parNode["dbname"].as<std::string>();
-			parSettings.port = parNode["port"].as<uint16_t>();
-			return true;
-		}
-	};
-} //namespace YAML
+#include <stdexcept>
+#include <sstream>
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range_core.hpp>
 
 namespace dinlib {
 	namespace {
 		std::string expand ( const char* parString );
+		std::string find_plugin_by_name ( std::vector<boost::string_ref>&& parSearchPaths, const std::string& parName );
+		void throw_if_plugin_failed ( const dindb::BackendPlugin& parPlugin, const std::string& parPluginPath, const std::string& parIntendedName );
 	} //unnamed namespace
 
-	bool load_settings (const std::string& parPath, dinlib::Settings& parOut, bool parExpand) {
+	void load_settings (const std::string& parPath, dinlib::Settings& parOut, bool parExpand) {
+		using dincore::split_and_trim;
+
 		const std::string path = (parExpand ? expand(parPath.c_str()) : parPath);
+		std::string search_paths;
 
-		try {
-			auto settings = YAML::LoadFile(path);
+		auto settings = YAML::LoadFile(path);
 
-			if (settings["db_settings"]) {
-				parOut.db = settings["db_settings"].as<dinlib::SettingsDB>();
-				return true;
-			}
+		if (not settings["backend_name"])
+			throw std::runtime_error("No backend_name given in the config file");
+		if (settings["backend_paths"])
+			search_paths += ":" + settings["backend_paths"].as<std::string>();
+		parOut.backend_name = settings["backend_name"].as<std::string>();
+		const std::string backend_settings_section = parOut.backend_name + "_settings";
+		if (settings[backend_settings_section]) {
+			auto settings_node = settings[backend_settings_section];
+			const std::string plugin_path = find_plugin_by_name(split_and_trim(search_paths, ':'), parOut.backend_name);
+			if (plugin_path.empty())
+				throw std::runtime_error(std::string("Unable to find any suitable plugin with the specified name \"") + parOut.backend_name + "\"");
+			parOut.backend_plugin = dindb::BackendPlugin(plugin_path, &settings_node);
+			throw_if_plugin_failed(parOut.backend_plugin, plugin_path, parOut.backend_name);
 		}
-		catch (const std::exception&) {
-			return false;
-		}
-
-		return false;
 	}
 
 	namespace {
@@ -83,5 +70,39 @@ namespace dinlib {
 			wordfree(&p);
 			return oss.str();
 		}
+
+		std::string find_plugin_by_name (std::vector<boost::string_ref>&& parSearchPaths, const std::string& parName) {
+			dincore::ShallowSearchPaths search_paths(std::move(parSearchPaths));
+			return search_paths.first_hit(
+				[&parName](boost::string_ref, const std::string& parPath) {
+					return dindb::backend_name(parPath) == parName;
+				},
+				dincore::SearchPaths::File
+			);
+		}
+
+		void throw_if_plugin_failed (const dindb::BackendPlugin& parPlugin, const std::string& parPluginPath, const std::string& parIntendedName) {
+			if (not parPlugin.is_loaded()) {
+				std::ostringstream oss;
+				oss << "Unable to load plugin \"" << parIntendedName <<
+					"\" found at path \"" << parPluginPath << '"';
+				throw std::runtime_error(oss.str());
+			}
+			if (parPlugin.name() != parIntendedName) {
+				std::ostringstream oss;
+				oss << "Plugin \"" << parIntendedName << "\" not found." <<
+					" Plugin at path \"" << parPluginPath << "\" reports \"" <<
+					parPlugin.name() << "\" as its name";
+				throw std::runtime_error(oss.str());
+			}
+			if (parPlugin.max_supported_interface_version() < parPlugin.backend_interface_version()) {
+				std::ostringstream oss;
+				oss << "Plugin \"" << parPlugin.name() << "\" at path \"" <<
+					parPluginPath << "\" uses interface version " << parPlugin.backend_interface_version() <<
+					" but the maximum supported interface version is " <<
+					parPlugin.max_supported_interface_version();
+				throw std::runtime_error(oss.str());
+			}
+		}
 	} //unnamed namespace
 } //namespace dinlib

src/core/CMakeLists.txt

@@ -0,0 +1,36 @@
+project(${bare_name}-core CXX)
+
+add_library(${PROJECT_NAME} SHARED
+	searchpaths.cpp
+	split_tags.cpp
+)
+
+target_link_libraries(${PROJECT_NAME}
+	PRIVATE ${bare_name}-if
+)
+
+target_include_directories(${PROJECT_NAME}
+	PRIVATE ${DINDEXER_PUB_INCLUDE_DIR}/${bare_name}-core
+)
+
+target_compile_features(${PROJECT_NAME}
+	INTERFACE cxx_nullptr
+	INTERFACE cxx_range_for
+	INTERFACE cxx_lambdas
+	INTERFACE cxx_decltype_auto
+	INTERFACE cxx_defaulted_functions
+	INTERFACE cxx_deleted_functions
+	INTERFACE cxx_auto_type
+	INTERFACE cxx_decltype_incomplete_return_types
+	INTERFACE cxx_defaulted_move_initializers
+	INTERFACE cxx_noexcept
+	INTERFACE cxx_rvalue_references
+	INTERFACE cxx_generalized_initializers
+	INTERFACE cxx_variadic_templates
+)
+
+install(TARGETS ${PROJECT_NAME}
+	LIBRARY DESTINATION lib
+	RUNTIME DESTINATION bin
+	ARCHIVE DESTINATION lib/static
+)

src/core/searchpaths.cpp

@@ -0,0 +1,136 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "searchpaths.hpp"
+#include <algorithm>
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range_core.hpp>
+#include <ciso646>
+#include <cassert>
+
+namespace dincore {
+	namespace {
+		const std::string& STR_to_string (const std::string& parString) {
+			return parString;
+		}
+
+		std::string STR_to_string (boost::string_ref parString) {
+			return std::string(parString.data(), parString.size());
+		}
+
+		template <typename STR>
+		std::string first_hit_impl (const std::vector<STR>& parPaths, SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType) {
+			using boost::filesystem::path;
+			using boost::filesystem::is_directory;
+			using boost::filesystem::directory_iterator;
+			using boost::filesystem::directory_entry;
+			using boost::make_iterator_range;
+
+			for (const auto& curr_dir_path : parPaths) {
+				path curr_path(STR_to_string(curr_dir_path));
+				auto listing = make_iterator_range(directory_iterator(curr_path), directory_iterator());
+				for (const directory_entry& entry : listing) {
+					if (
+						(parType == SearchPaths::Any) or
+						(parType == SearchPaths::Directory and is_directory(entry)) or
+						(parType == SearchPaths::File and not is_directory(entry))
+					) {
+						auto str_path = entry.path().string();
+						if (parPredicate(curr_dir_path, str_path))
+							return str_path;
+					}
+				}
+			}
+			return std::string();
+		}
+
+		//std::string make_file_path (boost::string_ref parPath, boost::string_ref parName) {
+		//	assert(not parName.empty());
+		//	if (parName.empty())
+		//		return std::string(parPath.data(), parPath.size());
+
+		//	std::string retval;
+		//	const std::size_t slash = (not parPath.empty() and parPath[parPath.size() - 1] != '/' ? 1 : 0);
+		//	retval.reserve(parPath.size() + parName.size() + slash);
+		//	std::copy(parPath.begin(), parPath.end(), retval.begin());
+		//	retval[parPath.size()] = '/';
+		//	std::copy(parName.begin(), parName.end(), retval.begin() + parPath.size() + slash);
+		//	return retval;
+		//}
+
+		bool is_same_filename (boost::string_ref parBaseDir, const std::string& parFullPath, boost::string_ref parFilename) {
+			assert(parBaseDir.size() < parFullPath.size());
+			return boost::string_ref(parFullPath).substr(parBaseDir.size() + 1) == parFilename;
+		}
+	} //unnamed namespace
+
+	SearchPaths::SearchPaths (std::vector<std::string>&& parList) :
+		m_paths(std::move(parList))
+	{
+	}
+
+	SearchPaths::SearchPaths (std::initializer_list<std::string> parInit) :
+		m_paths(parInit)
+	{
+	}
+
+	SearchPaths::~SearchPaths() noexcept = default;
+
+	void SearchPaths::add_path (std::string&& parPath) {
+		if (std::find(m_paths.begin(), m_paths.end(), parPath) == m_paths.end()) {
+			m_paths.emplace_back(std::move(parPath));
+		}
+	}
+
+	std::string SearchPaths::first_hit (boost::string_ref parFile, SearchType parType) const {
+		using std::placeholders::_1;
+		using std::placeholders::_2;
+		return first_hit_impl(m_paths, std::bind(&is_same_filename, _1, _2, parFile), parType);
+	}
+
+	std::string SearchPaths::first_hit (Predicate parPredicate, SearchType parType) const {
+		return first_hit_impl(m_paths, parPredicate, parType);
+	}
+
+	ShallowSearchPaths::ShallowSearchPaths (std::vector<boost::string_ref>&& parList) :
+		m_paths(std::move(parList))
+	{
+	}
+
+	ShallowSearchPaths::ShallowSearchPaths (std::initializer_list<boost::string_ref> parInit) :
+		m_paths(parInit)
+	{
+	}
+
+	ShallowSearchPaths::~ShallowSearchPaths() noexcept = default;
+
+	void ShallowSearchPaths::add_path (boost::string_ref parPath) {
+		if (std::find(m_paths.begin(), m_paths.end(), parPath) == m_paths.end()) {
+			m_paths.push_back(parPath);
+		}
+	}
+
+	std::string ShallowSearchPaths::first_hit (boost::string_ref parFile, SearchPaths::SearchType parType) const {
+		using std::placeholders::_1;
+		using std::placeholders::_2;
+		return first_hit_impl(m_paths, std::bind(&is_same_filename, _1, _2, parFile), parType);
+	}
+
+	std::string ShallowSearchPaths::first_hit (SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType) const {
+		return first_hit_impl(m_paths, parPredicate, parType);
+	}
+} //namespace dincore

src/core/split_tags.cpp

@@ -15,7 +15,7 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dindexer-common/split_tags.hpp"
+#include "dindexer-core/split_tags.hpp"
 #include <boost/algorithm/string/finder.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/trim.hpp>
@@ -24,8 +24,12 @@
 #include <boost/range/adaptor/filtered.hpp>
 #include <ciso646>
 
-namespace dinlib {
+namespace dincore {
 	std::vector<boost::string_ref> split_tags (const std::string& parCommaSeparatedList) {
+		return split_and_trim(parCommaSeparatedList, ',');
+	}
+
+	std::vector<boost::string_ref> split_and_trim (const std::string& parList, char parSeparator) {
 		using OutRange = boost::iterator_range<std::string::const_iterator>;
 		using boost::token_finder;
 		using boost::adaptors::transformed;
@@ -42,10 +46,10 @@ namespace dinlib {
 		//http://www.boost.org/doc/libs/1_60_0/doc/html/boost/algorithm/token_finder.html
 		//https://stackoverflow.com/questions/20781090/difference-between-boostsplit-vs-boostiter-split
 		return boost::copy_range<std::vector<string_ref>>(
-			iter_split(out_range, parCommaSeparatedList, token_finder([](char c){return ','==c;})) |
+			iter_split(out_range, parList, token_finder([parSeparator](char c){return parSeparator==c;})) |
 				transformed([](const OutRange& r){return trim_copy(r);}) |
 				transformed([](const OutRange& r){return string_ref(&*r.begin(), r.size());}) |
 				filtered([](const string_ref& r){return not r.empty();})
 		);
 	}
-} //namespace dinlib
+} //namespace dincore

src/delete/CMakeLists.txt

@@ -3,7 +3,6 @@ project(${bare_name}-delete CXX)
 add_executable(${PROJECT_NAME}
 	main.cpp
 	commandline.cpp
-	postgre_delete.cpp
 )
 
 target_include_directories(${PROJECT_NAME}

src/delete/main.cpp

@@ -18,7 +18,6 @@
 #include "commandline.hpp"
 #include "dindexer-common/settings.hpp"
 #include "dindexerConfig.h"
-#include "postgre_delete.hpp"
 #include <iostream>
 #include <ciso646>
 #include <ios>
@@ -26,7 +25,7 @@
 #include <cstdint>
 
 namespace {
-	bool confirm_delete (const din::IDDescMap& parMap) {
+	bool confirm_delete (const dindb::IDDescMap& parMap) {
 		for (const auto& itm : parMap) {
 			std::cout << "ID " << itm.first << '\t' << itm.second << '\n';
 		}
@@ -38,7 +37,7 @@ namespace {
 		return (answer.empty() or "y" == answer or "Y" == answer);
 	}
 
-	bool always_delete (const din::IDDescMap&) {
+	bool always_delete (const dindb::IDDescMap&) {
 		return true;
 	}
 } //unnamed namespace
@@ -62,12 +61,13 @@ int main (int parArgc, char* parArgv[]) {
 	}
 
 	dinlib::Settings settings;
-	{
-		const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings);
-		if (not loaded) {
-			std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n";
-			return 1;
-		}
+	try {
+		dinlib::load_settings(CONFIG_FILE_PATH, settings);
+	}
+	catch (const std::runtime_error& err) {
+		std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n";
+		std::cerr << err.what() << '\n';
+		return 1;
 	}
 
 	if (not vm.count("groupid")) {
@@ -77,7 +77,7 @@ int main (int parArgc, char* parArgv[]) {
 
 	const auto ids = vm["groupid"].as<std::vector<uint32_t>>();
 	auto confirm_func = (vm.count("confirm") ? &always_delete : &confirm_delete);
-	din::delete_group_from_db(settings.db, ids, confirm_func);
+	settings.backend_plugin.backend().delete_group(ids, confirm_func);
 	return 0;
 }
 

src/dindexerConfig.h.in

@@ -18,7 +18,7 @@
 #ifndef id48D6E1D45238460F99C2BCBFDE920791
 #define id48D6E1D45238460F99C2BCBFDE920791
 
-#include "dindexer-common/cmake_on_off.h"
+#include "duckhandy/cmake_on_off.h"
 
 #define PROGRAM_NAME "@bare_name@"
 #define VERSION_MAJOR @PROJECT_VERSION_MAJOR@

src/locate/CMakeLists.txt

@@ -3,7 +3,6 @@ project(${bare_name}-locate CXX)
 add_executable(${PROJECT_NAME}
 	main.cpp
 	commandline.cpp
-	postgre_locate.cpp
 	hash.cpp
 )
 

src/locate/main.cpp

@@ -16,9 +16,8 @@
  */
 
 #include "commandline.hpp"
-#include "postgre_locate.hpp"
 #include "dindexer-common/settings.hpp"
-#include "dindexer-common/split_tags.hpp"
+#include "dindexer-core/split_tags.hpp"
 #include "dindexerConfig.h"
 #include "hash.hpp"
 #include "glob2regex/glob2regex.hpp"
@@ -27,7 +26,7 @@
 #include <iterator>
 #include <algorithm>
 
-namespace din {
+namespace dindb {
 	std::ostream& operator<< (std::ostream& parStream, const LocatedItem& parItem) {
 		parStream << parItem.group_id << '\t' << parItem.id << '\t' << parItem.path;
 		return parStream;
@@ -41,14 +40,14 @@ namespace din {
 			'\t' << parItem.files_count << '\t' << dircount;
 		return parStream;
 	}
-} //namespace din
+} //namespace dindb
 
 namespace {
 	std::vector<boost::string_ref> extract_tags (const boost::program_options::variables_map& parVM) {
 		if (not parVM.count("tags"))
 			return std::vector<boost::string_ref>();
 		else
-			return dinlib::split_tags(parVM["tags"].as<std::string>());
+			return dincore::split_tags(parVM["tags"].as<std::string>());
 	}
 } //unnamed namespace
 
@@ -72,31 +71,33 @@ int main (int parArgc, char* parArgv[]) {
 	}
 
 	dinlib::Settings settings;
-	{
-		const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings);
-		if (not loaded) {
-			std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n";
-			return 1;
-		}
+	try {
+		dinlib::load_settings(CONFIG_FILE_PATH, settings);
+	}
+	catch (const std::runtime_error& err) {
+		std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n";
+		std::cerr << err.what() << '\n';
+		return 1;
 	}
 
+	auto& db = settings.backend_plugin.backend();
 	if (vm.count("set")) {
-		const auto results = din::locate_sets_in_db(settings.db, vm["substring"].as<std::string>(), not not vm.count("case-insensitive"));
-		std::copy(results.begin(), results.end(), std::ostream_iterator<din::LocatedSet>(std::cout, "\n"));
+		const auto results = db.locate_sets_in_db(vm["substring"].as<std::string>(), not not vm.count("case-insensitive"));
+		std::copy(results.begin(), results.end(), std::ostream_iterator<dindb::LocatedSet>(std::cout, "\n"));
 	}
 	else {
-		std::vector<din::LocatedItem> results;
+		std::vector<dindb::LocatedItem> results;
 		const std::vector<boost::string_ref> tags = extract_tags(vm);
 
 		if (vm.count("byhash")) {
 			const auto hash = din::hash(vm["substring"].as<std::string>());
-			results = din::locate_in_db(settings.db, hash, tags);
+			results = db.locate_in_db(hash, tags);
 		}
 		else {
 			const auto search_regex = g2r::convert(vm["substring"].as<std::string>(), not vm.count("case-insensitive"));
-			results = din::locate_in_db(settings.db, search_regex, tags);
+			results = db.locate_in_db(search_regex, tags);
 		}
-		std::copy(results.begin(), results.end(), std::ostream_iterator<din::LocatedItem>(std::cout, "\n"));
+		std::copy(results.begin(), results.end(), std::ostream_iterator<dindb::LocatedItem>(std::cout, "\n"));
 	}
 	return 0;
 }

src/locate/postgre_locate.hpp

@@ -1,53 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef id1AE05A59AE0E4A4490040FD85D9AF665
-#define id1AE05A59AE0E4A4490040FD85D9AF665
-
-#include "dindexer-common/settings.hpp"
-#include <vector>
-#include <string>
-#include <cstdint>
-#include <boost/utility/string_ref.hpp>
-
-namespace mchlib {
-	struct TigerHash;
-} //namespace mchlib
-
-namespace din {
-	struct LocatedItem {
-		std::string path;
-		uint64_t id;
-		uint32_t group_id;
-	};
-
-	struct LocatedSet {
-		std::string desc;
-		uint32_t id;
-		uint32_t files_count;
-		uint32_t dir_count;
-	};
-
-	using TagList = std::vector<boost::string_ref>;
-
-	std::vector<LocatedItem> locate_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, const TagList& parTags );
-	std::vector<LocatedItem> locate_in_db ( const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags );
-	std::vector<LocatedSet> locate_sets_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, bool parCaseInsensitive );
-	std::vector<LocatedSet> locate_sets_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive );
-} //namespace din
-
-#endif

src/machinery/CMakeLists.txt

@@ -26,9 +26,8 @@ add_library(${PROJECT_NAME} SHARED
 	scantask/setbasic.cpp
 )
 
-target_include_directories(${PROJECT_NAME}
+target_include_directories(${PROJECT_NAME} SYSTEM
 	PRIVATE ${MAGIC_INCLUDE_DIR}
-	PRIVATE ${CMAKE_CURRENT_BINARY_DIR}
 )
 
 target_link_libraries(${PROJECT_NAME}
@@ -37,6 +36,7 @@ target_link_libraries(${PROJECT_NAME}
 )
 
 target_include_directories(${PROJECT_NAME}
+	PRIVATE ${CMAKE_CURRENT_BINARY_DIR}
 	PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
 	PUBLIC ${CMAKE_SOURCE_DIR}/lib/better-enums
 )

src/machinery/discinfo.cpp

@@ -17,7 +17,7 @@
 
 #include "discinfo.hpp"
 #include "pathname.hpp"
-#include "helpers/lengthof.h"
+#include "duckhandy/lengthof.h"
 #include <map>
 #include <fstream>
 #include <boost/tokenizer.hpp>
@@ -35,6 +35,10 @@
 #	include <iostream>
 #endif
 #include <cstdint>
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range_core.hpp>
+
+namespace fs = boost::filesystem;
 
 namespace mchlib {
 	namespace {
@@ -167,6 +171,41 @@ namespace mchlib {
 			return static_cast<LinuxDeviceTypes>(retval);
 		}
 #endif
+
+		std::string find_with_same_inode (const std::string& parWhat, const char* parWhere) {
+			using boost::make_iterator_range;
+
+			struct stat st;
+			if (stat(parWhat.c_str(), &st))
+				return std::string();
+
+			const auto& inode = st.st_ino;
+
+			fs::path p(parWhere);
+			if (not fs::exists(p) or not fs::is_directory(p)) {
+				throw std::runtime_error(
+					std::string("Search path \"") + p.string() +
+					"\" doesn't exist");
+			}
+
+			for (const fs::directory_entry& itm : make_iterator_range(fs::directory_iterator(p), fs::directory_iterator())) {
+				struct stat curr_st;
+				if (not stat(itm.path().c_str(), &curr_st) and inode == curr_st.st_ino)
+					return fs::basename(itm);
+			}
+
+			return std::string();
+		}
+
+		//Get disc label by doing the equivalent of:
+		//find -L /dev/disk/by-label -inum $(stat -c %i /dev/sda1) -print
+		std::string retrieve_label (const std::string& parDev) {
+			return find_with_same_inode(parDev, "/dev/disk/by-label");
+		}
+
+		std::string retrieve_uuid (const std::string& parDev) {
+			return find_with_same_inode(parDev, "/dev/disk/by-uuid");
+		}
 	} //unnamed namespace
 
 	DiscInfo::DiscInfo (std::string&& parPath) :
@@ -187,6 +226,11 @@ namespace mchlib {
 			}
 			input_path.pop_right();
 		} while(input_path.atom_count() > 0);
+
+		if (mountpoint_found()) {
+			m_label = retrieve_label(m_device);
+			m_uuid = retrieve_uuid(m_device);
+		}
 	}
 
 	bool DiscInfo::mountpoint_found() const {
@@ -329,4 +373,12 @@ namespace mchlib {
 		};
 	}
 #endif
+
+	const std::string& DiscInfo::label() const {
+		return m_label;
+	}
+
+	const std::string& DiscInfo::filesystem_uuid() const {
+		return m_uuid;
+	}
 } //namespace mchlib

src/machinery/discinfo.hpp

@@ -52,11 +52,15 @@ namespace mchlib {
 		OpticalTypes optical_type ( void ) const;
 		DriveTypes drive_type ( void ) const;
 #endif
+		const std::string& label ( void ) const;
+		const std::string& filesystem_uuid ( void ) const;
 
 	private:
 		const std::string m_initial_path;
 		std::string m_mountpoint;
 		std::string m_device;
+		std::string m_label;
+		std::string m_uuid;
 	};
 } //namespace mchlib
 

src/machinery/machineryConfig.h.in

@@ -18,7 +18,7 @@
 #ifndef id32A00B72E29D463795CBF66517387E8D
 #define id32A00B72E29D463795CBF66517387E8D
 
-#include "dindexer-common/cmake_on_off.h"
+#include "duckhandy/cmake_on_off.h"
 
 #if CMAKE_@DINDEXER_WITH_NICE_MEDIA_TYPES@
 #	define WITH_NICE_MEDIA_TYPES

src/machinery/machinery_info.cpp

@@ -17,7 +17,7 @@
 
 #include "dindexer-machinery/machinery_info.hpp"
 #include "dindexerConfig.h"
-#include "helpers/stringize.h"
+#include "duckhandy/stringize.h"
 
 namespace mchlib {
 	boost::string_ref lib_signature() {

src/machinery/scantask/dirtree.cpp

@@ -18,7 +18,7 @@
 #include "dindexer-machinery/scantask/dirtree.hpp"
 #include "dindexer-machinery/recorddata.hpp"
 #include "dindexer-machinery/set_listing.hpp"
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 #include "filesearcher.hpp"
 #include "pathname.hpp"
 #include <utility>

src/machinery/scantask/mediatype.cpp

@@ -21,6 +21,7 @@
 #	include "dindexer-machinery/mediatype.hpp"
 #endif
 #include "dindexer-machinery/recorddata.hpp"
+#include "discinfo.hpp"
 #include <utility>
 
 namespace mchlib {
@@ -57,6 +58,12 @@ namespace mchlib {
 #else
 			data.type = media_type_to_char(m_default);
 #endif
+
+			{
+				DiscInfo info((std::string(m_search_path)));
+				data.disk_label = info.label();
+				data.fs_uuid = info.filesystem_uuid();
+			}
 		}
 	} //namespace scantask
 } //namespace mchlib

src/machinery/scantask/setbasic.cpp

@@ -29,13 +29,15 @@ namespace mchlib {
 		}
 
 		void SetBasic::on_data_destroy (SetRecordDataFull& parData) {
-			static_cast<SetRecordData&>(parData).name.clear();
 			parData.name.clear();
+			parData.fs_uuid.clear();
+			parData.disk_label.clear();
 		}
 
 		void SetBasic::on_data_create (SetRecordDataFull& parData) {
 			parData.name = m_set_name;
-			static_cast<SetRecordData&>(parData).name = parData.name;
+			parData.fs_uuid.clear();
+			parData.disk_label.clear();
 		}
 	} //namespace scantask
 } //namespace mchlib

src/machinery/tiger.cpp

@@ -16,6 +16,7 @@
  */
 
 #include "dindexer-machinery/tiger.hpp"
+#include "duckhandy/lexical_cast.hpp"
 #include <fstream>
 #include <cstdint>
 #include <memory>
@@ -24,6 +25,7 @@
 #include <utility>
 #include <sstream>
 #include <iomanip>
+#include <boost/utility/string_ref.hpp>
 
 extern "C" void tiger ( const char* parStr, uint64_t parLength, uint64_t parHash[3], char parPadding );
 
@@ -153,6 +155,20 @@ namespace mchlib {
 		return oss.str();
 	}
 
+	TigerHash string_to_tiger (const std::string& parString) {
+		using boost::string_ref;
+		using TigerPartType = decltype(TigerHash::part_a);
+
+		assert(parString.size() == sizeof(TigerHash) * 2);
+
+		TigerHash retval;
+		const string_ref inp(parString);
+		retval.part_a = swap_long(dhandy::lexical_cast<TigerPartType, dhandy::tags::hex>(inp.substr(0, sizeof(TigerPartType) * 2)));
+		retval.part_b = swap_long(dhandy::lexical_cast<TigerPartType, dhandy::tags::hex>(inp.substr(sizeof(TigerPartType) * 2, sizeof(TigerPartType) * 2)));
+		retval.part_c = swap_long(dhandy::lexical_cast<TigerPartType, dhandy::tags::hex>(inp.substr(sizeof(TigerPartType) * 4, sizeof(TigerPartType) * 2)));
+		return retval;
+	}
+
 	void tiger_data (const std::string& parData, TigerHash& parHash) {
 		tiger (parData.data(), parData.size(), parHash.data, g_tiger_padding);
 	}

src/main/CMakeLists.txt

@@ -12,7 +12,6 @@ add_executable(${PROJECT_NAME}
 
 make_timestamp(${bare_name} timestamp.h.in)
 target_include_directories(${PROJECT_NAME}
-	PRIVATE ${CMAKE_SOURCE_DIR}/include
 	PRIVATE ${CMAKE_SOURCE_DIR}/lib/pbl/pbl/src/src
 	PRIVATE ${CMAKE_CURRENT_BINARY_DIR}
 )

src/main/damerau_levenshtein.h

@@ -18,7 +18,7 @@
 #ifndef id88738025C6B24BDEB604A5AE3C36EE8D
 #define id88738025C6B24BDEB604A5AE3C36EE8D
 
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 #if defined(__cplusplus)
 #	include <cstddef>
 #else

src/main/findactions.c

@@ -17,7 +17,7 @@
 
 #include "findactions.h"
 #include "dindexerConfig.h"
-#include "helpers/lengthof.h"
+#include "duckhandy/lengthof.h"
 #include <dirent.h>
 #include <string.h>
 #include <stdlib.h>

src/main/main.c

@@ -19,7 +19,7 @@
 
 #include "dindexerConfig.h"
 #include "findactions.h"
-#include "helpers/lengthof.h"
+#include "duckhandy/lengthof.h"
 #include "builtin_feats.h"
 #include "damerau_levenshtein.h"
 #include <string.h>

src/main/timestamp.h.in

@@ -18,7 +18,7 @@
 #ifndef id4A9E94ACFA43498097637025A5BA69E5
 #define id4A9E94ACFA43498097637025A5BA69E5
 
-#include "dindexer-common/cmake_on_off.h"
+#include "duckhandy/cmake_on_off.h"
 
 #if CMAKE_@DINDEXER_WITH_BUILD_DATE@
 #	define DINDEXER_BUILD_DATE "@DINDEXER_BUILD_DATE@"

src/main/utf8_ops.h

@@ -20,7 +20,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
-#include "helpers/compatibility.h"
+#include "duckhandy/compatibility.h"
 
 typedef uint32_t Character;
 

src/navigate/AutomemBase.hpp

@@ -1,91 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef idDB299AE3079F4830BB1D543C07F7B594
-#define idDB299AE3079F4830BB1D543C07F7B594
-
-//Classes in this file are low level and only intended to be used by library
-//code. They don't do much by themselves and shoud never be used from outside.
-
-#include <memory>
-#include <algorithm>
-#include <cassert>
-#include <cstdint>
-
-#if !defined(NDEBUG)
-#	define ASSERTIONSENABLED
-#endif
-
-namespace din {
-	const size_t MAX_STACK_ALLOC_SIZE = 128;
-
-	///-------------------------------------------------------------------------
-	///Heap-based allocation, only gets raw memory
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A=std::allocator<T> >
-	class AutomemRawBase_heap {
-	protected:
-		AutomemRawBase_heap ( void );
-		AutomemRawBase_heap ( const AutomemRawBase_heap& ) = delete; //Copy ctor can't be implemented at this level
-		AutomemRawBase_heap ( AutomemRawBase_heap&& parOther );
-
-		template <typename... Args>
-		T* GetNewT ( size_t parIndex, Args&&... parArgs );
-		T* AllocMemory ( void );
-		void FreeMemory ( void ) noexcept;
-		void swap ( AutomemRawBase_heap& parOther ) noexcept { std::swap(m_localMem, parOther.m_localMem); }
-	private:
-		void operator= (const AutomemRawBase_heap&);
-
-#if defined(ASSERTIONSENABLED)
-		typedef uintptr_t PTR_INT_TYPE;
-		static_assert(sizeof(PTR_INT_TYPE) == sizeof(char*), "Wrong uintptr_t size");
-#endif
-		T* m_localMem;
-	};
-
-	///-------------------------------------------------------------------------
-	///Stack-based allocation, only gets raw memory
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S>
-	class AutomemRawBase_stack {
-	protected:
-		AutomemRawBase_stack ( void ) = default;
-		AutomemRawBase_stack ( const AutomemRawBase_stack& ) = delete; //Copy ctor can't be implemented at this level
-		AutomemRawBase_stack ( AutomemRawBase_stack&& ) = delete;
-		~AutomemRawBase_stack ( void ) = default;
-
-		template <typename... Args>
-		T* GetNewT ( size_t parIndex, Args&&... parArgs );
-		T* AllocMemory ( void );
-		void FreeMemory ( void ) noexcept { return; }
-	private:
-#if defined(ASSERTIONSENABLED)
-		typedef uintptr_t PTR_INT_TYPE;
-		static_assert(sizeof(PTR_INT_TYPE) == sizeof(char*), "Wrong uintptr_t size");
-#endif
-		void operator= (const AutomemRawBase_stack&);
-		typename std::aligned_storage<sizeof(T), alignof(T)>::type m_localMem[S];
-	};
-} //namespace din
-
-#include "AutomemBase.inl"
-
-#if defined(ASSERTIONSENABLED)
-#	undef ASSERTIONSENABLED
-#endif
-#endif

src/navigate/AutomemBase.inl

@@ -1,118 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-namespace din {
-	namespace {
-#if defined(ASSERTIONSENABLED)
-		const char g_guard = 0xAB;
-#endif
-	} //unnamed namespace
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	AutomemRawBase_heap<T, S, A>::AutomemRawBase_heap() {
-#if !defined(NDEBUG)
-		m_localMem = nullptr;
-#endif
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	AutomemRawBase_heap<T, S, A>::AutomemRawBase_heap (AutomemRawBase_heap&& parOther) {
-#if !defined(NDEBUG)
-		m_localMem = nullptr;
-#endif
-		this->swap(parOther);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	T* AutomemRawBase_heap<T, S, A>::AllocMemory() {
-#if !defined(NDEBUG)
-		Assert(nullptr == m_localMem);
-#endif
-		m_localMem = A().allocate(S);
-#if defined(ASSERTIONSENABLED)
-		assert(reinterpret_cast<PTR_INT_TYPE>(m_localMem) % alignof(T) == 0); //Make sure alignment is correct
-		std::fill(
-			reinterpret_cast<char*>(&m_localMem[0]),
-			reinterpret_cast<char*>(&m_localMem[0]) + sizeof(m_localMem),
-			g_guard
-		);
-#endif
-		return m_localMem;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void AutomemRawBase_heap<T, S, A>::FreeMemory() noexcept {
-#if !defined(NDEBUG)
-		Assert(nullptr != m_localMem);
-#endif
-		A().deallocate(m_localMem, S);
-#if !defined(NDEBUG)
-		m_localMem = nullptr;
-#endif
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	template <typename... Args>
-	T* AutomemRawBase_heap<T, S, A>::GetNewT (size_t parIndex, Args&&... parArgs) {
-		assert(parIndex < S);
-		T* const location = m_localMem + parIndex;
-#if defined(ASSERTIONSENABLED)
-		assert(reinterpret_cast<PTR_INT_TYPE>(location) % alignof(T) == 0);
-		assert(g_guard == *reinterpret_cast<const char*>(location));
-#endif
-		return new(location) T(std::forward<Args...>(parArgs)...);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S>
-	T* AutomemRawBase_stack<T, S>::AllocMemory() {
-#if defined(ASSERTIONSENABLED)
-		assert(reinterpret_cast<PTR_INT_TYPE>(m_localMem) % alignof(T) == 0); //Make sure alignment is correct
-		std::fill(
-			reinterpret_cast<char*>(&m_localMem[0]),
-			reinterpret_cast<char*>(&m_localMem[0]) + sizeof(m_localMem),
-			g_guard
-		);
-#endif
-		return reinterpret_cast<T*>(&m_localMem[0]);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S>
-	template <typename... Args>
-	T* AutomemRawBase_stack<T, S>::GetNewT (size_t parIndex, Args&&... parArgs) {
-		assert(parIndex < S);
-		auto* const location = &m_localMem[parIndex];
-#if defined(ASSERTIONSENABLED)
-		assert(reinterpret_cast<PTR_INT_TYPE>(location) % alignof(T) == 0);
-		assert(g_guard == *reinterpret_cast<const char*>(location));
-#endif
-		return new(location) T(std::forward<Args...>(parArgs)...);
-	}
-} //namespace din

src/navigate/CMakeLists.txt

@@ -5,7 +5,6 @@ add_executable(${PROJECT_NAME}
 	commandline.cpp
 	commandprocessor.cpp
 	entrypath.cpp
-	dbsource.cpp
 	linereader.cpp
 	listdircontent.cpp
 )

src/navigate/IteratorOnPtr.hpp

@@ -1,314 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef id1B822B1775424D879AA6AD1739B5BC91
-#define id1B822B1775424D879AA6AD1739B5BC91
-
-#include <iterator>
-#include <cstdint>
-#include <type_traits>
-#include <cassert>
-
-namespace din {
-	namespace implem_iop {
-		template <typename P, typename T, int32_t I>
-		class IteratorOnPtr_base : public std::iterator<std::random_access_iterator_tag, T> {
-			typedef std::iterator<std::random_access_iterator_tag, T> parent_type;
-		public:
-			//Typedefs to be STL-compliant
-			typedef typename parent_type::iterator_category iterator_category;
-			typedef typename parent_type::value_type value_type;
-			typedef typename parent_type::difference_type difference_type;
-			typedef typename parent_type::pointer pointer;
-			typedef typename std::remove_reference<T>::type& reference;
-
-			IteratorOnPtr_base ( P parPointer, difference_type parSize );
-			IteratorOnPtr_base ( const IteratorOnPtr_base& parOther );
-			template <typename P1, typename T1>
-			explicit IteratorOnPtr_base ( const IteratorOnPtr_base<P1, T1, I>& parOther );
-			~IteratorOnPtr_base ( void );
-
-			difference_type operator- ( const IteratorOnPtr_base& parOther ) const;
-
-			bool operator== ( const IteratorOnPtr_base& parOther ) const { return m_pointer == parOther.m_pointer; }
-			bool operator!= ( const IteratorOnPtr_base& parOther ) const { return not operator==(parOther); }
-			bool operator< ( const IteratorOnPtr_base& parOther ) const { return m_pointer < parOther.m_pointer; }
-			bool operator>= ( const IteratorOnPtr_base& parOther ) const { return not operator<(parOther); }
-			bool operator> ( const IteratorOnPtr_base& parOther ) const { return parOther < *this; }
-			bool operator<= ( const IteratorOnPtr_base& parOther ) const { return not operator>(parOther); }
-
-			P GetPointer ( void ) { return m_pointer; }
-			const P GetPointer ( void ) const { return m_pointer; }
-#if !defined(NDEBUG)
-			difference_type GetSize ( void ) const { return GetSize(0); }
-#endif
-
-		protected:
-			enum {
-				STEP = (I < 0 ? -I : I)
-			};
-
-#if !defined(NDEBUG)
-			difference_type GetSize ( difference_type parAdvance ) const;
-			bool CanAdvance ( difference_type parAdvance ) const;
-			void AlterSize ( difference_type parAdvance );
-#endif
-			void MoveIterator ( difference_type parAdvance );
-
-			P m_pointer;
-
-		private:
-#if !defined(NDEBUG)
-			difference_type m_size;
-#endif
-		};
-	} //namespace implem_iop
-
-	template <typename P, typename T, bool R>
-	class IteratorOnPtr : public implem_iop::IteratorOnPtr_base<P, T, (R ? -1 : 1)> {
-		typedef implem_iop::IteratorOnPtr_base<P, T, (R ? -1 : 1)> parent_type;
-		enum {
-			STEP = parent_type::STEP
-		};
-	public:
-		typedef typename parent_type::iterator_category iterator_category;
-		typedef typename parent_type::value_type value_type;
-		typedef typename parent_type::difference_type difference_type;
-		typedef typename parent_type::pointer pointer;
-		typedef typename parent_type::reference reference;
-
-		IteratorOnPtr ( void );
-		IteratorOnPtr ( P parPointer, difference_type parSize );
-		IteratorOnPtr ( const IteratorOnPtr& parOther ) : parent_type(parOther) { return; }
-		template <typename P1, typename T1>
-		IteratorOnPtr ( const IteratorOnPtr<P1, T1, R>& parOther ) : parent_type(parOther) {}
-		~IteratorOnPtr ( void );
-
-		IteratorOnPtr& operator++ ( void ); //pre
-		IteratorOnPtr operator++ ( int ); //post
-		IteratorOnPtr& operator-- ( void );
-		IteratorOnPtr operator-- ( int );
-		reference operator* ( void );
-		pointer operator-> ( void );
-
-		using parent_type::operator-;
-		IteratorOnPtr operator+ ( difference_type parOther ) const;
-		IteratorOnPtr operator- ( difference_type parOther ) const;
-		IteratorOnPtr& operator+= ( difference_type parOther );
-		IteratorOnPtr& operator-= ( difference_type parOther );
-	protected:
-	private:
-	};
-
-	namespace implem_iop {
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-#if !defined(NDEBUG)
-		IteratorOnPtr_base<P, T, I>::IteratorOnPtr_base (P parPointer, difference_type parSize) :
-			m_pointer(parPointer),
-			m_size(parSize)
-#else
-		IteratorOnPtr_base<P, T, I>::IteratorOnPtr_base (P parPointer, difference_type) :
-			m_pointer(parPointer)
-#endif
-		{
-			static_assert(I != 0, "Step must be non-zero");
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		template <typename P1, typename T1>
-		IteratorOnPtr_base<P, T, I>::IteratorOnPtr_base (const IteratorOnPtr_base<P1, T1, I>& parOther) {
-			m_pointer = parOther.GetPointer();
-#if !defined(NDEBUG)
-			m_size = parOther.GetSize();
-#endif
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		IteratorOnPtr_base<P, T, I>::IteratorOnPtr_base (const IteratorOnPtr_base& parOther) {
-			m_pointer = parOther.m_pointer;
-#if !defined(NDEBUG)
-			m_size = parOther.m_size;
-#endif
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		IteratorOnPtr_base<P, T, I>::~IteratorOnPtr_base() {
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		typename IteratorOnPtr_base<P, T, I>::difference_type IteratorOnPtr_base<P, T, I>::operator- (const IteratorOnPtr_base& parOther) const {
-			if (I > 0)
-				return m_pointer - parOther.m_pointer;
-			else
-				return parOther.m_pointer - m_pointer;
-		}
-
-#if !defined(NDEBUG)
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		typename IteratorOnPtr_base<P, T, I>::difference_type IteratorOnPtr_base<P, T, I>::GetSize (difference_type parAdvance) const {
-			return m_size - STEP * parAdvance;
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		bool IteratorOnPtr_base<P, T, I>::CanAdvance (difference_type parAdvance) const {
-			return (m_size >= STEP * parAdvance);
-		}
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		void IteratorOnPtr_base<P, T, I>::AlterSize (difference_type parAdvance) {
-			m_size = GetSize(parAdvance);
-		}
-#endif
-
-		///---------------------------------------------------------------------
-		///---------------------------------------------------------------------
-		template <typename P, typename T, int32_t I>
-		void IteratorOnPtr_base<P, T, I>::MoveIterator (difference_type parAdvance) {
-#if !defined(NDEBUG)
-			assert(CanAdvance(parAdvance));
-			AlterSize(parAdvance);
-#endif
-			m_pointer += I * parAdvance;
-		}
-	} //namespace implem_iop
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>::IteratorOnPtr() :
-		parent_type(NULL, 0)
-	{
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>::IteratorOnPtr (P parPointer, difference_type parSize) :
-		parent_type(parPointer, parSize)
-	{
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>::~IteratorOnPtr() {
-	}
-
-	///-------------------------------------------------------------------------
-	///Pre-increment.
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>& IteratorOnPtr<P, T, R>::operator++() {
-		this->MoveIterator(1);
-		return *this;
-	}
-
-	///-------------------------------------------------------------------------
-	///Post-increment.
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R> IteratorOnPtr<P, T, R>::operator++ (int) {
-		IteratorOnPtr<P, T, R> retVal(*this);
-		this->MoveIterator(1);
-		return retVal;
-	}
-
-	///-------------------------------------------------------------------------
-	///Pre-decrement.
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>& IteratorOnPtr<P, T, R>::operator--() {
-		this->MoveIterator(-1);
-		return *this;
-	}
-
-	///-------------------------------------------------------------------------
-	///Post-decrement.
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R> IteratorOnPtr<P, T, R>::operator-- (int) {
-		IteratorOnPtr<P, T, R> retVal(*this);
-		this->MoveIterator(-1);
-		return retVal;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	typename IteratorOnPtr<P, T, R>::reference IteratorOnPtr<P, T, R>::operator*() {
-		return *(this->m_pointer);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	typename IteratorOnPtr<P, T, R>::pointer IteratorOnPtr<P, T, R>::operator->() {
-		return this->m_pointer;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R> IteratorOnPtr<P, T, R>::operator+ (difference_type parOther) const {
-		IteratorOnPtr<P, T, R> retVal(*this);
-		retVal += parOther;
-		return retVal;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R> IteratorOnPtr<P, T, R>::operator- (difference_type parOther) const {
-		IteratorOnPtr<P, T, R> retVal(*this);
-		retVal -= parOther;
-		return retVal;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>& IteratorOnPtr<P, T, R>::operator+= (difference_type parOther) {
-		this->MoveIterator(parOther);
-		return *this;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename P, typename T, bool R>
-	IteratorOnPtr<P, T, R>& IteratorOnPtr<P, T, R>::operator-= (difference_type parOther) {
-		this->MoveIterator(-parOther);
-		return *this;
-	}
-} //namespace din
-
-#endif

src/navigate/MaxSizedArray.hpp

@@ -1,114 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef idBC9F804ADD33468A9C7657E823FFC706
-#define idBC9F804ADD33468A9C7657E823FFC706
-
-#include "AutomemBase.hpp"
-#include "IteratorOnPtr.hpp"
-#include <memory>
-#include <cstddef>
-#include <type_traits>
-#include <cassert>
-#include <utility>
-
-namespace din {
-	const size_t MAXSZARR_MAX_STACK_ALLOC_SIZE = MAX_STACK_ALLOC_SIZE;
-
-	//TODO: add a template parameter to force stack allocation regardless of
-	//sizeof(T) * S
-	template <typename T, size_t S, typename A=std::allocator<T> >
-	class MaxSizedArray :
-		private std::conditional<
-			(sizeof(T) * S > (MAXSZARR_MAX_STACK_ALLOC_SIZE > 4 * sizeof(T*) ? MAXSZARR_MAX_STACK_ALLOC_SIZE - sizeof(T*) : MAXSZARR_MAX_STACK_ALLOC_SIZE)),
-			AutomemRawBase_heap<T, S, A>,
-			AutomemRawBase_stack<T, S>
-		>::type
-	{
-		typedef typename std::conditional<
-			(sizeof(T) * S > (MAXSZARR_MAX_STACK_ALLOC_SIZE > 4 * sizeof(T*) ? MAXSZARR_MAX_STACK_ALLOC_SIZE - sizeof(T*) : MAXSZARR_MAX_STACK_ALLOC_SIZE)),
-			AutomemRawBase_heap<T, S, A>,
-			AutomemRawBase_stack<T, S>
-		>::type parent_type;
-
-	public:
-		typedef IteratorOnPtr<T*, T, false> iterator;
-		typedef IteratorOnPtr<const T*, const T, false> const_iterator;
-		typedef IteratorOnPtr<T*, T, true> reverse_iterator;
-		typedef IteratorOnPtr<const T*, const T, true> const_reverse_iterator;
-		typedef T& reference;
-		typedef T&& mov_reference;
-		typedef const T& const_reference;
-		typedef size_t size_type;
-		typedef ptrdiff_t difference_type;
-		typedef T value_type;
-		typedef T* pointer;
-		typedef const T* const_pointer;
-
-		enum MAX_SIZE_ENUM {
-			MAX_SIZE = S
-		};
-
-		MaxSizedArray ( void );
-		MaxSizedArray ( const MaxSizedArray& parOther );
-		MaxSizedArray ( MaxSizedArray&& parOther );
-		~MaxSizedArray ( void );
-
-		bool empty ( void ) const;
-		size_type size ( void ) const;
-		size_type capacity ( void ) const { return S; }
-		size_type max_size ( void ) const { return S; }
-		void push_back ( const value_type& parNewItem );
-		void push_back ( value_type&& parNewItem );
-		void pop_back ( void );
-		iterator erase ( const iterator& parDele );
-		iterator erase ( const iterator& parFrom, const iterator& parToExcl );
-		void clear ( void );
-		mov_reference operator[] ( size_type parIndex );
-		const_reference operator[] ( size_type parIndex ) const;
-		MaxSizedArray& operator= ( const MaxSizedArray& parOther );
-		bool operator== ( const MaxSizedArray& parOther ) const;
-		bool operator!= ( const MaxSizedArray& parOther ) const;
-		pointer GetPointer ( void );
-		const_pointer GetPointer ( void ) const;
-		void reserve ( size_type parReserve );
-
-		iterator begin ( void );
-		const_iterator begin ( void ) const;
-		iterator end ( void );
-		const_iterator end ( void ) const;
-		reverse_iterator rbegin ( void );
-		const_reverse_iterator rbegin ( void ) const;
-		reverse_iterator rend ( void );
-		const_reverse_iterator rend ( void ) const;
-		reference front ( void ) { return (*this)[0]; }
-		reference back ( void ) { return (*this)[size() - 1]; }
-		const_reference front ( void ) const { return (*this)[0]; }
-		const_reference back ( void ) const { return (*this)[size() - 1]; }
-
-	private:
-		pointer GetPointer_NoAssert ( void ) { return m_localMem; }
-		const_pointer GetPointer_NoAssert ( void ) const { return m_localMem; }
-
-		T* m_localMem; //A copy of memory pointer is required
-		size_type m_used;
-	};
-} //namespace din
-
-#include "MaxSizedArray.inl"
-
-#endif

src/navigate/MaxSizedArray.inl

@@ -1,281 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-namespace din {
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	MaxSizedArray<T, S, A>::MaxSizedArray() {
-		m_localMem = this->AllocMemory();
-		m_used = 0;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	MaxSizedArray<T, S, A>::MaxSizedArray (const MaxSizedArray& parOther) :
-		parent_type(),
-		m_used(0)
-	{
-		m_localMem = this->AllocMemory();
-		const size_type count = parOther.size();
-		for (size_type z = 0; z < count; ++z) {
-			this->push_back(parOther[z]);
-		}
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	MaxSizedArray<T, S, A>::MaxSizedArray (MaxSizedArray&& parOther) :
-		parent_type(),
-		m_used(0)
-	{
-		m_localMem = this->AllocMemory();
-		const size_type count = parOther.size();
-		for (size_type z = 0; z < count; ++z) {
-			this->push_back(std::move(parOther[z]));
-		}
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	MaxSizedArray<T, S, A>::~MaxSizedArray() {
-		this->clear();
-		this->FreeMemory();
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	bool MaxSizedArray<T, S, A>::empty() const {
-		return 0 == m_used;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::size_type MaxSizedArray<T, S, A>::size() const {
-		return m_used;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void MaxSizedArray<T, S, A>::push_back (value_type&& parNewItem) {
-		assert(size() < capacity());
-		this->GetNewT(m_used, std::move(parNewItem));
-		++m_used;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void MaxSizedArray<T, S, A>::push_back (const value_type& parNewItem) {
-		assert(size() < capacity());
-		this->GetNewT(m_used, parNewItem);
-		++m_used;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void MaxSizedArray<T, S, A>::pop_back() {
-		assert(not empty());
-		m_localMem[m_used - 1].~T();
-		--m_used;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::mov_reference MaxSizedArray<T, S, A>::operator[] (size_type parIndex) {
-		assert(parIndex < size());
-		return std::move(m_localMem[parIndex]);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_reference MaxSizedArray<T, S, A>::operator[] (size_type parIndex) const {
-		assert(parIndex < size());
-		return m_localMem[parIndex];
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	MaxSizedArray<T, S, A>& MaxSizedArray<T, S, A>::operator= (const MaxSizedArray& parOther) {
-		m_used = parOther.m_used;
-		std::copy(parOther.GetMemPtr(), parOther.GetMemPtr() + parOther.size(), this->GetMemPtr());
-		return *this;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	bool MaxSizedArray<T, S, A>::operator== (const MaxSizedArray& parOther) const {
-		if (size() != parOther.size())
-			return false;
-		for (size_type z = 0; z < size(); ++z) {
-			if ((*this)[z] != parOther[z])
-				return false;
-		}
-		return true;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	bool MaxSizedArray<T, S, A>::operator!= (const MaxSizedArray& parOther) const {
-		return not (*this == parOther);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::pointer MaxSizedArray<T, S, A>::GetPointer() {
-		assert(size() > 0);
-		return GetPointer_NoAssert();
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_pointer MaxSizedArray<T, S, A>::GetPointer() const {
-		assert(size() > 0);
-		return GetPointer_NoAssert();
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void MaxSizedArray<T, S, A>::clear() {
-		const size_type count = this->size();
-		for (size_type z = 0; z < count; ++z) {
-			(*this)[z].~T();
-		}
-		m_used = 0;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::iterator MaxSizedArray<T, S, A>::begin() {
-		return iterator(GetPointer_NoAssert(), size());
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_iterator MaxSizedArray<T, S, A>::begin() const {
-		return const_iterator(GetPointer_NoAssert(), size());
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::iterator MaxSizedArray<T, S, A>::end() {
-		return iterator(GetPointer_NoAssert() + size(), 0);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_iterator MaxSizedArray<T, S, A>::end() const {
-		return const_iterator(GetPointer_NoAssert() + size(), 0);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::reverse_iterator MaxSizedArray<T, S, A>::rbegin() {
-		return reverse_iterator(GetPointer_NoAssert() + size() - 1, size());
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_reverse_iterator MaxSizedArray<T, S, A>::rbegin() const {
-		return const_reverse_iterator(GetPointer_NoAssert() + size() - 1, size());
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::reverse_iterator MaxSizedArray<T, S, A>::rend() {
-		return reverse_iterator(GetPointer_NoAssert() - 1, 0);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::const_reverse_iterator MaxSizedArray<T, S, A>::rend() const {
-		return const_reverse_iterator(GetPointer_NoAssert() - 1, 0);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::iterator MaxSizedArray<T, S, A>::erase (const iterator& parDele) {
-		assert(end() != parDele);
-		return erase(parDele, parDele + 1);
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	typename MaxSizedArray<T, S, A>::iterator MaxSizedArray<T, S, A>::erase (const iterator& parFrom, const iterator& parToExcl) {
-		assert(parFrom >= begin());
-		assert(parToExcl <= end());
-		assert(parToExcl >= parFrom);
-
-		//I'm doing this in two passes: first, I'm deleting as many elements as
-		//the ones that would be left at the end (that is, delete and move),
-		//then delete everything to the end of the buffer, if necessary.
-		const size_type deleCount = static_cast<size_type>(parToExcl - parFrom);
-		const size_type firstIndexToDele = static_cast<size_type>(parFrom - begin());
-		const size_type& sz = m_used;
-		assert(firstIndexToDele + deleCount <= sz);
-		const size_type deleAndCopyCount = sz - (firstIndexToDele + deleCount);
-
-		//As said, make room and copy from the cut tail
-		for (size_type z = firstIndexToDele; z < firstIndexToDele + deleAndCopyCount; z++) {
-			(*this)[z].~T();
-			new(&(*this)[z]) T((*this)[z + deleCount]);
-		}
-
-		//Any leftover is rubbish
-		for (size_type z = firstIndexToDele + deleAndCopyCount; z < sz; z++) {
-			(*this)[z].~T();
-		}
-
-		m_used -= deleCount;
-		return begin() + firstIndexToDele;
-	}
-
-	///-------------------------------------------------------------------------
-	///-------------------------------------------------------------------------
-	template <typename T, size_t S, typename A>
-	void MaxSizedArray<T, S, A>::reserve (size_type parReserve) {
-		assert(parReserve <= S);
-		if (parReserve > S) {
-			throw std::length_error("Unable to reserve more memory than the build-time size for MaxSizedArray");
-		}
-	}
-} //namespace din

src/navigate/dbsource.cpp

@@ -1,183 +0,0 @@
-/* Copyright 2015, 2016, Michele Santullo
- * This file is part of "dindexer".
- *
- * "dindexer" is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * "dindexer" is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "dbsource.hpp"
-#include "dindexer-common/settings.hpp"
-#include "pq/connection.hpp"
-#include "helpers/infix_iterator.hpp"
-#include <ciso646>
-#include <utility>
-#include <boost/lexical_cast.hpp>
-#include <sstream>
-#include <boost/range/algorithm/copy.hpp>
-
-namespace din {
-	namespace {
-		const uint32_t g_files_query_limit = 500;
-
-		std::ostream& operator<< (std::ostream& parOut, const std::vector<std::string>& parCols) {
-			parOut << '"';
-			boost::copy(parCols, infix_ostream_iterator<std::string>(parOut, "\", \""));
-			parOut << '"';
-			return parOut;
-		}
-	} //unnamed namespace
-
-	const DBSource::SetDetailsMap DBSource::m_set_details_map {
-		{SetDetail_Desc, "desc"},
-		{SetDetail_Type, "type"},
-		{SetDetail_CreeationDate, "creation"},
-		{SetDetail_AppName, "app_name"},
-		{SetDetail_ID, "id"}
-	};
-	const DBSource::FileDetailsMap DBSource::m_file_details_map {
-		{FileDetail_ID, "id"},
-		{FileDetail_Path, "path"},
-		{FileDetail_Level, "level"},
-		{FileDetail_GroupID, "group_id"},
-		{FileDetail_IsDir, "is_directory"},
-		{FileDetail_IsSymLink, "is_symlink"},
-		{FileDetail_Size, "size"},
-		{FileDetail_Hash, "hash"},
-		{FileDetail_IsHashValid, "is_hash_valid"},
-		{FileDetail_ATime, "access_time"},
-		{FileDetail_MTime, "modify_time"},
-		{FileDetail_Unreadable, "unreadable"},
-		{FileDetail_MimeType, "mimetype"},
-		{FileDetail_Charset, "charset"}
-	};
-
-	struct DBSource::LocalData {
-		explicit LocalData ( const dinlib::SettingsDB& parDBSettings ) :
-			conn(
-				std::string(parDBSettings.username),
-				std::string(parDBSettings.password),
-				std::string(parDBSettings.dbname),
-				std::string(parDBSettings.address),
-				parDBSettings.port
-			)
-		{
-		}
-
-		pq::Connection conn;
-	};
-
-	DBSource::DBSource (const dinlib::SettingsDB& parDBSettings) :
-		m_local_data(new LocalData(parDBSettings))
-	{
-		assert(not m_local_data->conn.is_connected());
-	}
-
-	DBSource::~DBSource() noexcept {
-	}
-
-	void DBSource::disconnect() {
-		if (m_local_data->conn.is_connected()) {
-			m_local_data->conn.disconnect();
-		}
-	}
-
-	pq::Connection& DBSource::get_conn() {
-		if (not m_local_data->conn.is_connected()) {
-			m_local_data->conn.connect();
-		}
-		return m_local_data->conn;
-	}
-
-	std::vector<uint32_t> DBSource::sets() {
-		using boost::lexical_cast;
-
-		auto& conn = get_conn();
-		const std::string query = "SELECT \"id\" FROM \"sets\";";
-		auto res = conn.query(query);
-		std::vector<uint32_t> retval;
-
-		retval.reserve(res.size());
-		for (const auto& row : res) {
-			retval.push_back(lexical_cast<uint32_t>(row[0]));
-		}
-		return retval;
-	}
-
-	void DBSource::query_no_conditions (const ColumnList& parCols, boost::string_ref parTable, const std::vector<uint32_t>& parIDs, std::function<void(std::string&&)> parCallback) {
-		std::ostringstream oss;
-		oss << "SELECT " << parCols << ' ' <<
-			"FROM \"" << parTable << "\" " <<
-			"WHERE \"id\"=ANY($1) " <<
-			"ORDER BY \"desc\" ASC " <<
-			"LIMIT " << g_files_query_limit << ';';
-
-		auto& conn = get_conn();
-		auto result = conn.query(oss.str(), parIDs);
-		for (auto row : result) {
-			for (auto val : row) {
-				parCallback(std::move(val));
-			}
-		}
-	}
-
-	void DBSource::query_files_in_dir (const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, uint32_t parGroupID, QueryCallback parCallback) {
-		std::ostringstream oss;
-		oss << "SELECT " << parCols << ' ' <<
-			"FROM \"files\" WHERE " <<
-			"\"level\"=$1 " <<
-			"AND \"group_id\"=$2 " <<
-			"AND str_begins_with(\"path\", COALESCE($3, '')) " <<
-			"ORDER BY \"is_directory\" DESC, \"path\" ASC " <<
-			"LIMIT " << g_files_query_limit << ';';
-
-		auto& conn = get_conn();
-		auto result = conn.query(
-			oss.str(),
-			parLevel,
-			parGroupID,
-			parDir
-		);
-		for (auto row : result) {
-			for (auto val : row) {
-				parCallback(std::move(val));
-			}
-		}
-	}
-
-	std::vector<std::string> DBSource::paths_starting_by (uint32_t parGroupID, uint16_t parLevel, boost::string_ref parPath) {
-		std::ostringstream oss;
-		oss << "SELECT \"path\" ||\n" <<
-			"(SELECT CASE \"is_directory\"\n" <<
-			"WHEN TRUE THEN '/'\n" <<
-			"ELSE ''\n" <<
-			"END) as path FROM \"files\" WHERE \"group_id\"=$1 AND " <<
-			"\"level\"=$2 AND str_begins_with(\"path\", COALESCE($3, '')) " <<
-			"ORDER BY \"is_directory\" DESC, \"path\" ASC LIMIT " <<
-			g_files_query_limit << ';';
-
-		auto& conn = get_conn();
-		auto result = conn.query(
-			oss.str(),
-			parGroupID,
-			parLevel,
-			parPath
-		);
-		std::vector<std::string> retval;
-		retval.reserve(retval.size());
-		for (auto row : result) {
-			assert(not row.empty());
-			retval.push_back(row[0]);
-		}
-		return retval;
-	}
-} //namespace din

src/navigate/entrypath.cpp

@@ -16,7 +16,7 @@
  */
 
 #include "entrypath.hpp"
-#include "helpers/infix_iterator.hpp"
+#include "duckhandy/infix_iterator.hpp"
 #include <boost/spirit/include/qi_core.hpp>
 #include <boost/spirit/include/qi_parse.hpp>
 #include <boost/spirit/include/qi_lit.hpp>

src/navigate/listdircontent.cpp

@@ -17,8 +17,8 @@
 
 #include "listdircontent.hpp"
 #include "entrypath.hpp"
-#include "dbsource.hpp"
-#include "helpers/infix_iterator.hpp"
+#include "backends/db_backend.hpp"
+#include "duckhandy/infix_iterator.hpp"
 #include <cassert>
 #include <utility>
 #include <boost/range/algorithm/copy.hpp>
@@ -68,7 +68,7 @@ namespace din {
 		}
 	} //unnamed namespace
 
-	ListDirContent::ListDirContent (DBSource* parDB) :
+	ListDirContent::ListDirContent (dindb::Backend* parDB) :
 		m_cache(g_max_cached_lists),
 		m_db(parDB)
 	{
@@ -85,14 +85,14 @@ namespace din {
 
 		//Requested item is not cached, so we need to query the db now
 		if (parDir.points_to_group()) {
-			auto sets_ids = m_db->sets();
-			auto sets_info = m_db->set_details<SetDetail_ID, SetDetail_Desc, SetDetail_CreeationDate>(sets_ids);
+			auto sets_ids = m_db->find_all_sets();
+			auto sets_info = m_db->find_set_details(sets_ids);
 			m_cache.push_back(std::make_pair(curr_path, db_result_to_vec(sets_info)));
 		}
 		else {
 			auto path_prefix = parDir.file_path();
 			const auto set_id = parDir.group_id();
-			auto files_info = m_db->file_details<FileDetail_Path>(set_id, parDir.level() + 1, path_prefix);
+			auto files_info = m_db->find_file_details(set_id, parDir.level() + 1, path_prefix);
 			m_cache.push_back(std::make_pair(curr_path, db_result_to_vec(files_info)));
 		}
 		return last_cached_item(curr_path);
@@ -115,7 +115,7 @@ namespace din {
 		else {
 			const auto set_id = parDir.group_id();
 			const auto path_prefix = parDir.file_path();
-			auto file_list = m_db->paths_starting_by(set_id, parDir.level(), path_prefix);
+			auto file_list = m_db->find_paths_starting_by(set_id, parDir.level(), path_prefix);
 			for (auto& file_item : file_list) {
 				file_item = EntryPath(file_item)[return_level];
 			}

src/navigate/listdircontent.hpp

@@ -23,15 +23,18 @@
 #include <string>
 #include <vector>
 
+namespace dindb {
+	class Backend;
+} //namespace dindb
+
 namespace din {
 	class EntryPath;
-	class DBSource;
 
 	class ListDirContent {
 		using ListType = std::vector<std::string>;
 		using CachedItemType = std::pair<std::string, ListType>;
 	public:
-		explicit ListDirContent ( DBSource* parDB );
+		explicit ListDirContent ( dindb::Backend* parDB );
 		~ListDirContent ( void ) noexcept = default;
 
 		const ListType& ls ( const EntryPath& parDir ) const;
@@ -41,7 +44,7 @@ namespace din {
 		const ListType& last_cached_item ( const std::string& parCurrPath ) const;
 
 		mutable boost::circular_buffer<CachedItemType> m_cache;
-		DBSource* m_db;
+		dindb::Backend* m_db;
 	};
 } //namespace din
 

src/navigate/main.cpp

@@ -19,7 +19,6 @@
 #include "commandprocessor.hpp"
 #include "dindexer-common/settings.hpp"
 #include "entrypath.hpp"
-#include "dbsource.hpp"
 #include "dindexerConfig.h"
 #include "linereader.hpp"
 #include "listdircontent.hpp"
@@ -27,11 +26,10 @@
 #include <ciso646>
 #include <string>
 #include <vector>
-#include <cassert>
 #include <boost/range/algorithm/copy.hpp>
 
 namespace {
-	void do_navigation ( din::DBSource& parDB );
+	void do_navigation ( dindb::Backend& parDB );
 
 	bool on_exit ( void );
 	void on_pwd ( const din::EntryPath& parDirMan );
@@ -53,17 +51,16 @@ int main (int parArgc, char* parArgv[]) {
 	}
 
 	dinlib::Settings settings;
-	{
-		const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings);
-		if (not loaded) {
-			std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n";
-			return 1;
-		}
+	try {
+		dinlib::load_settings(CONFIG_FILE_PATH, settings);
+	}
+	catch (const std::runtime_error& err) {
+		std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n";
+		std::cerr << err.what() << '\n';
+		return 1;
 	}
 
-	din::DBSource db_source(settings.db);
-
-	do_navigation(db_source);
+	do_navigation(settings.backend_plugin.backend());
 	return 0;
 }
 
@@ -81,7 +78,7 @@ namespace {
 		boost::copy(ls_result, std::ostream_iterator<std::string>(std::cout, "\n"));
 	}
 
-	void do_navigation (din::DBSource& parDB) {
+	void do_navigation (dindb::Backend& parDB) {
 		const std::string prompt;
 		din::ListDirContent ls(&parDB);
 		din::LineReader lines(&ls);
@@ -92,7 +89,7 @@ namespace {
 		din::EntryPath dir_man;
 		proc.add_command("exit", &on_exit, 0);
 		proc.add_command("cd", std::function<void(const std::string&)>(std::bind(&din::EntryPath::push_piece, &dir_man, std::placeholders::_1)), 1);
-		proc.add_command("disconnect", std::function<void()>(std::bind(&din::DBSource::disconnect, &parDB)), 0);
+		proc.add_command("disconnect", std::function<void()>(std::bind(&dindb::Backend::disconnect, std::ref(parDB))), 0);
 		proc.add_command("pwd", std::function<void()>(std::bind(&on_pwd, std::ref(dir_man))), 0);
 		proc.add_command("ls", std::function<void()>(std::bind(on_ls, std::ref(ls), std::ref(dir_man))), 0);
 		do {

src/pq/CMakeLists.txt

@@ -1,5 +1,8 @@
 project(${bare_name}-pq CXX)
 
+set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fPIC")
+set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fPIC")
+
 add_library(${PROJECT_NAME} STATIC
 	connection.cpp
 	databaseexception.cpp
@@ -9,10 +12,12 @@ add_library(${PROJECT_NAME} STATIC
 
 target_include_directories(${PROJECT_NAME}
 	PRIVATE ${libpqtypes_INCLUDE_DIRS}
+	PRIVATE ${PostgreSQL_INCLUDE_DIRS}
 )
 target_link_libraries(${PROJECT_NAME}
 	PRIVATE ${bare_name}-if
 	PRIVATE pqtypes
+	PRIVATE ${PostgreSQL_LIBRARIES}
 )
 
 target_compile_features(${PROJECT_NAME}

src/scan/CMakeLists.txt

@@ -4,7 +4,6 @@ include(WithMediaAutodetect)
 
 add_executable(${PROJECT_NAME}
 	main.cpp
-	dbbackend.cpp
 	commandline.cpp
 )