diff --git a/.gitmodules b/.gitmodules index 94e1d37..dfd4a5f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,9 @@ [submodule "lib/better-enums"] path = lib/better-enums url = https://github.com/aantron/better-enums +[submodule "lib/duckhandy"] + path = lib/duckhandy + url = https://github.com/KingDuckZ/duckhandy.git +[submodule "lib/incredis"] + path = lib/incredis + url = ../incredis.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e556a1d..c9b1531 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ include(gccversion) include(CPack) include(CTest) include(timestamp) +include(shared_git_project) option(DINDEXER_DEBUG_CFG_FILE "Enable to set the config file path to the build path" OFF) option(DINDEXER_WITH_MEDIA_AUTODETECT "Enable code that tries to autodetect the media type and sets --type automatically" ON) @@ -57,7 +58,7 @@ if ("${DINDEXER_CONFIG_FILE}" STREQUAL "") endif() message(STATUS "Config file set to \"${DINDEXER_CONFIG_FILE}\"") -find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options) +find_package(Boost 1.53.0 REQUIRED COMPONENTS program_options filesystem system) find_package(PostgreSQL 8.3 REQUIRED) find_package(YamlCpp 0.5.1 REQUIRED) import_libpqtypes_project("${PostgreSQL_INCLUDE_DIRS}" "-O3 ${march_flag}") @@ -87,7 +88,6 @@ configure_file( target_include_directories(${PROJECT_NAME} SYSTEM INTERFACE ${Boost_INCLUDE_DIRS} - INTERFACE ${PostgreSQL_INCLUDE_DIRS} ) target_compile_features(${PROJECT_NAME} @@ -128,29 +128,35 @@ add_subdirectory(src/common) add_subdirectory(src/machinery) add_subdirectory(lib/pbl) add_subdirectory(lib/glob2regex) +add_subdirectory(src/backends) +add_subdirectory(src/core) +add_shared_git_project(lib/duckhandy) +add_shared_git_project(lib/incredis EXCLUDE_FROM_ALL) #Actions add_subdirectory(src/main) add_subdirectory(src/scan) add_subdirectory(src/delete) -add_subdirectory(src/query) +#add_subdirectory(src/query) add_subdirectory(src/locate) add_subdirectory(src/navigate) add_subdirectory(src/tag) #Tests if (BUILD_TESTING) - add_subdirectory(test/gtest) + add_shared_git_project(test/gtest) add_subdirectory(test/unit) add_subdirectory(test/unit_cli) endif() target_link_libraries(${PROJECT_NAME} - INTERFACE ${PostgreSQL_LIBRARIES} INTERFACE ${Boost_LIBRARIES} INTERFACE ${bare_name}-pq INTERFACE ${bare_name}-inc ) +target_link_libraries(${bare_name}-inc + INTERFACE duckhandy +) target_compile_definitions(${PROJECT_NAME} INTERFACE WITH_PROGRESS_FEEDBACK diff --git a/README.md b/README.md index ab02929..11fff34 100644 --- a/README.md +++ b/README.md @@ -168,8 +168,6 @@ and the following libraries: * **machinery** the bulk of dindexer, such as the indexing functionality * **pq** a c++ wrapper for libpq (PostgreSql) -Additionally inside include/helpers/ you will find header-only implementations of very generic helper tools. - Currently, actions are just the command line frontend to access the functionalities in the libraries. This way writing a new CLI or GUI should be relatively easy. This structure might change in the future. diff --git a/cmake/Modules/shared_git_project.cmake b/cmake/Modules/shared_git_project.cmake new file mode 100644 index 0000000..54463ef --- /dev/null +++ b/cmake/Modules/shared_git_project.cmake @@ -0,0 +1,110 @@ +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# For more information, please refer to + +cmake_minimum_required(VERSION 2.8.12.2 FATAL_ERROR) + +function (add_shared_git_project SUBMODULE_PATH) + if (IS_ABSOLUTE "${SUBMODULE_PATH}") + set(submod_path "${SUBMODULE_PATH}") + else() + set(submod_path "${CMAKE_CURRENT_SOURCE_DIR}/${SUBMODULE_PATH}") + endif() + + if (NOT EXISTS "${submod_path}") + message(FATAL_ERROR "Path \"${submod_path}\" doesn't exist") + endif() + if (NOT IS_DIRECTORY "${submod_path}") + message(FATAL_ERROR "Path \"${submod_path}\" is not a valid directory") + endif() + + if (NOT EXISTS "${submod_path}/.git") + message(FATAL_ERROR ".git not found in \"${submod_path}\". Not a git submodule?") + endif() + if (IS_DIRECTORY "${submod_path}/.git") + message(FATAL_ERROR "\"${submod_path}.git\" is a directory, not a file as expected. Not a git submodule?") + endif() + + get_filename_component(proj_name_orig "${submod_path}" NAME) + if ("${proj_name_orig}" STREQUAL "") + message(FATAL_ERROR "Couldn't make up a name for given project in \"${submod_path}\"") + endif() + + string(MAKE_C_IDENTIFIER "${proj_name_orig}" proj_name_c_id) + string(TOUPPER ${proj_name_c_id} proj_name) + + get_property(shared_projects_list GLOBAL PROPERTY SHARED_PROJECTS_LIST) + list(FIND shared_projects_list ${proj_name} found_index) + if (${found_index} GREATER -1) + #nothing to do, the variable is already set so the project must have been + #included already + return() + endif() + + #Obtain the path to the working tree + # see http://stackoverflow.com/questions/27379818/git-possible-to-use-same-submodule-working-copy-by-multiple-projects + # git rev-parse --git-dir --show-toplevel + execute_process( + COMMAND git rev-parse --show-toplevel + WORKING_DIRECTORY "${submod_path}" + OUTPUT_VARIABLE matched_gitdir + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + #Make sure we got an absolute path + if (IS_ABSOLUTE "${matched_gitdir}") + set(reported_submodule_dir "${matched_gitdir}") + else() + file(RELATIVE_PATH reported_submodule_dir "${CMAKE_CURRENT_SOURCE_DIR}" "${submod_path}/${matched_gitdir}") + endif() + unset(matched_gitdir) + + #Check if submodule is a subdirectory of the current source dir + file(RELATIVE_PATH reported_submodule_rel_path "${CMAKE_CURRENT_SOURCE_DIR}" "${reported_submodule_dir}") + string(LENGTH "${reported_submodule_rel_path}" rel_path_len) + if (${rel_path_len} GREATER 2) + string(SUBSTRING "${reported_submodule_rel_path}" 0 3 first_bit) + if ("../" STREQUAL "${first_bit}") + set(is_out_of_dirtree ON) + else() + set(is_out_of_dirtree OFF) + endif() + unset(first_bit) + else() + set(is_out_of_dirtree OFF) + endif() + unset(rel_path_len) + + #Globally mark current submodule as handled + set_property(GLOBAL APPEND PROPERTY SHARED_PROJECTS_LIST ${proj_name}) + + set(shared_project_binary "${CMAKE_CURRENT_BINARY_DIR}/shared_projects/${proj_name_orig}") + if (is_out_of_dirtree) + #message(FATAL_ERROR "Would call add_subdirectory(\"${reported_submodule_dir}\" \"${shared_project_binary}\")") + add_subdirectory("${reported_submodule_dir}" "${shared_project_binary}") + else() + #message(FATAL_ERROR "Would call add_subdirectory(\"${reported_submodule_rel_path}\")") + add_subdirectory("${reported_submodule_rel_path}") + endif() +endfunction() diff --git a/dindexer.sql.in b/dindexer.sql.in index 26e0b0b..c2d893a 100644 --- a/dindexer.sql.in +++ b/dindexer.sql.in @@ -5,7 +5,7 @@ -- Dumped from database version 9.5.2 -- Dumped by pg_dump version 9.5.2 --- Started on 2016-05-18 00:47:01 CEST +-- Started on 2016-06-03 20:30:32 CEST SET statement_timeout = 0; SET lock_timeout = 0; @@ -160,6 +160,8 @@ CREATE TABLE sets ( creation timestamp with time zone DEFAULT now() NOT NULL, app_name character varying NOT NULL, content_type character(1) DEFAULT 'G'::bpchar NOT NULL, + disk_label text NOT NULL, + fs_uuid text NOT NULL, CONSTRAINT chk_sets_type CHECK (((type = 'C'::bpchar) OR (type = 'D'::bpchar) OR (type = 'V'::bpchar) OR (type = 'B'::bpchar) OR (type = 'F'::bpchar) OR (type = 'H'::bpchar) OR (type = 'Z'::bpchar) OR (type = 'O'::bpchar))) ); @@ -309,7 +311,7 @@ GRANT ALL ON SCHEMA public TO postgres; GRANT ALL ON SCHEMA public TO PUBLIC; --- Completed on 2016-05-18 00:47:01 CEST +-- Completed on 2016-06-03 20:30:33 CEST -- -- PostgreSQL database dump complete diff --git a/dindexer.yml b/dindexer.yml index 1d3b7a9..eeb1a26 100644 --- a/dindexer.yml +++ b/dindexer.yml @@ -1,8 +1,21 @@ %YAML 1.2 --- -db_settings: - username: your_username - password: your_password - dbname: dindexer - port: 5432 - address: 200.100.200.100 +backend_name: postgresql +postgresql_settings: + connection: + username: your_username + password: your_password + dbname: dindexer + port: 5432 + address: 200.100.200.100 + +redis_settings: + connection: + address: 200.100.200.100 + port: 6379 + + script_paths: + - /home/duckz/dev/build/dindexer/Debug/src/backends/redis/lua + - /home/duckz/dev/build/dindexer/Release/src/backends/redis/lua + +backend_paths: path to your build_dir/src/backend diff --git a/docs/hiredis_libev_multithread.md b/docs/hiredis_libev_multithread.md new file mode 100644 index 0000000..c86602f --- /dev/null +++ b/docs/hiredis_libev_multithread.md @@ -0,0 +1,166 @@ +[TOC] + +# Using hiredis+libev in a separate thread for events + +Used sources: + +1. [stackoverflow.com/questions/14621261][link_so_1] +2. [stackoverflow.com/questions/8611126][link_so_2] +3. [libev thread locking example][link_ev_doc] + + +## Explanation + +### Link 1 + +Possibly due to the original question asked, [Link 1][link_so_1] seems to show that you need to have at least two ev_loop objects created in order to run one in a separate thread. The code on that page: + +```cpp +//This program is demo for using pthreads with libev. +//Try using Timeout values as large as 1.0 and as small as 0.000001 +//and notice the difference in the output + +//(c) 2009 debuguo +//(c) 2013 enthusiasticgeek for stack overflow +//Free to distribute and improve the code. Leave credits intact + +#include +#include // for puts +#include +#include + +pthread_mutex_t lock; +double timeout = 0.00001; +ev_timer timeout_watcher; +int timeout_count = 0; + +ev_async async_watcher; +int async_count = 0; + +struct ev_loop* loop2; + +void* loop2thread(void* args) +{ + printf("Inside loop 2"); // Here one could initiate another timeout watcher + ev_loop(loop2, 0); // similar to the main loop - call it say timeout_cb1 + return NULL; +} + +static void async_cb (EV_P_ ev_async *w, int revents) +{ + //puts ("async ready"); + pthread_mutex_lock(&lock); //Don't forget locking + ++async_count; + printf("async = %d, timeout = %d \n", async_count, timeout_count); + pthread_mutex_unlock(&lock); //Don't forget unlocking +} + +static void timeout_cb (EV_P_ ev_timer *w, int revents) // Timer callback function +{ + //puts ("timeout"); + if (ev_async_pending(&async_watcher)==false) { //the event has not yet been processed (or even noted) by the event loop? (i.e. Is it serviced? If yes then proceed to) + ev_async_send(loop2, &async_watcher); //Sends/signals/activates the given ev_async watcher, that is, feeds an EV_ASYNC event on the watcher into the event loop. + } + + pthread_mutex_lock(&lock); //Don't forget locking + ++timeout_count; + pthread_mutex_unlock(&lock); //Don't forget unlocking + w->repeat = timeout; + ev_timer_again(loop, &timeout_watcher); //Start the timer again. +} + +int main (int argc, char** argv) +{ + if (argc < 2) { + puts("Timeout value missing.\n./demo "); + return -1; + } + timeout = atof(argv[1]); + + struct ev_loop *loop = EV_DEFAULT; //or ev_default_loop (0); + + //Initialize pthread + pthread_mutex_init(&lock, NULL); + pthread_t thread; + + // This loop sits in the pthread + loop2 = ev_loop_new(0); + + //This block is specifically used pre-empting thread (i.e. temporary interruption and suspension of a task, without asking for its cooperation, with the intention to resume that task later.) + //This takes into account thread safety + ev_async_init(&async_watcher, async_cb); + ev_async_start(loop2, &async_watcher); + pthread_create(&thread, NULL, loop2thread, NULL); + + ev_timer_init (&timeout_watcher, timeout_cb, timeout, 0.); // Non repeating timer. The timer starts repeating in the timeout callback function + ev_timer_start (loop, &timeout_watcher); + + // now wait for events to arrive + ev_loop(loop, 0); + //Wait on threads for execution + pthread_join(thread, NULL); + + pthread_mutex_destroy(&lock); + return 0; +} +``` + +with the comment *"Note for libev 4+ ev_loop should be ev_run."* is still slightly useful but it shouldn't be taken as a model. + + +### Link 2 + +The [second link][link_so_2] is what shows that one ev_loop is enough. This is the code provided by the original poster: + +```cpp + void RedisSubscriber::Start() { + m_redis = redisAsyncConnect(m_addr.c_str(),m_port); + m_redis->data = (void*)this; + + m_loop = ev_loop_new(EVFLAG_NOINOTIFY); + redisLibevAttach(m_loop, m_redis); + redisAsyncSetConnectCallback(m_redis,connectCallback); + redisAsyncSetDisconnectCallback(m_redis,disconnectCallback); + redisAsyncCommand(m_redis, subscribeCallback, NULL, "SUBSCRIBE %s", m_channel.c_str()); + + m_thread = boost::thread(ev_loop,m_loop,0); + } + + void RedisSubscriber::Stop() { + redisAsyncFree(m_redis); + m_thread.join(); + m_redis = 0; + } + + void RedisSubscriber::connectCallback(const redisAsyncContext *c) { + + } + + void RedisSubscriber::disconnectCallback(const redisAsyncContext *c, int status) { + RedisSubscriber* r = (RedisSubscriber*)(c->data); + ev_unloop(r->m_loop,EVUNLOOP_ALL); + } + + void RedisSubscriber::subscribeCallback(redisAsyncContext *c, void *r, void *privdata) { + + } +``` + +There are no accepted answers, but the answer from *themoondothshine* provides very useful info. Here is what it says: + +Assuming that you mean ev_run for your boost::thread, here's what you can do: + +1. Setup an `ev_async` +2. In the callback of `ev_async` call `ev_break`. +3. Call `ev_async_send` from `RedisSubscriber::Stop()`. `ev_async` watchers are thread-safe -- it uses memory barriers for synchronising between threads. + +This will cause the event loop to stop, and `m_thread.join()` will return. + + +### Link 3 + +The [THREAD LOCKING EXAMPLE][link_ev_doc] shows how to lock in order to protect the ev_loop object in use. + +[link_so_1]: http://stackoverflow.com/questions/14621261/using-libev-with-multiple-threads#14779930 +[link_so_2]: http://stackoverflow.com/questions/8611126/hiredis-libev-and-boostthreads +[link_ev_doc]: http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod#THREAD_LOCKING_EXAMPLE diff --git a/docs/posts/04_whats_cooking.md b/docs/posts/04_whats_cooking.md new file mode 100644 index 0000000..2a084d8 --- /dev/null +++ b/docs/posts/04_whats_cooking.md @@ -0,0 +1,11 @@ +### What's cooking? ### + +It's been roughly a month since I released version 0.1.5b and most of the activity has been on the [dev branch](https://bitbucket.org/King_DuckZ/dindexer/branch/dev) since then. It's still a bit too early to make a new release, but if you want to get a peek at the new features this is the right moment as commit e2275ce should give you a working version of #dindexer. + +Most notably, you will have a `tag` command, that will allow you to tag files. You should also be able to list files by tag, using the `dindexer locate --tags ` command. + +The less evident change (although that's what took up most of my time) is the separation of the PostgreSQL code from the CL client. You have been asking for Sqlite support, I heard you and I'm getting everything ready so that an Sqlite plugin can be implemented! Currently, PostgreSQL is still the only backend you can choose, so please be patient... or even better, step in and contribute to dindexer! :) + +[bitbucket.org/King_DuckZ/dindexer](https://bitbucket.org/King_DuckZ/dindexer) + +\#dindexer #linux #opensource #cpp #sqlite diff --git a/flat_git.yml b/flat_git.yml new file mode 100644 index 0000000..bf79c81 --- /dev/null +++ b/flat_git.yml @@ -0,0 +1,3 @@ +inplace_submodules: + - pbl + - better-enums diff --git a/include/backends/backend_loader.hpp b/include/backends/backend_loader.hpp new file mode 100644 index 0000000..7795d0c --- /dev/null +++ b/include/backends/backend_loader.hpp @@ -0,0 +1,62 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id756A258A98B24B0DB2529BCEEC5137E2 +#define id756A258A98B24B0DB2529BCEEC5137E2 + +#include +#include +#include + +namespace YAML { + class Node; +} //namespace YAML + +namespace dindb { + class Backend; + + using BackendPtr = std::unique_ptr; + + class BackendPlugin { + public: + BackendPlugin ( void ); + BackendPlugin ( BackendPlugin&& ) = default; + BackendPlugin ( const std::string& parSOPath, const YAML::Node* parConfig ); + ~BackendPlugin ( void ) noexcept; + + const boost::string_ref& name ( void ) const; + Backend& backend ( void ); + const Backend& backend ( void ) const; + bool is_loaded ( void ) const; + int backend_interface_version ( void ) const; + int max_supported_interface_version ( void ) const; + + BackendPlugin& operator= ( BackendPlugin&& ) = default; + + private: + using SoHandle = std::unique_ptr; + + SoHandle m_lib; + BackendPtr m_backend; + boost::string_ref m_name; + int m_iface_ver; + }; + + std::string backend_name ( const std::string& parSOPath ); +} //namespace dindb + +#endif diff --git a/include/dindexer-common/cmake_on_off.h b/include/backends/backend_version.hpp similarity index 65% rename from include/dindexer-common/cmake_on_off.h rename to include/backends/backend_version.hpp index 165ab95..8b31e82 100644 --- a/include/dindexer-common/cmake_on_off.h +++ b/include/backends/backend_version.hpp @@ -15,24 +15,11 @@ * along with "dindexer". If not, see . */ -#ifndef id9B6B373E88404330ADEE51A4EC861787 -#define id9B6B373E88404330ADEE51A4EC861787 +#ifndef idAA27B58429DB41C2AF53204CC5010E94 +#define idAA27B58429DB41C2AF53204CC5010E94 -#if !defined(CMAKE_ON) -# define CMAKE_ON 1 -# define CMAKE_on 1 -# define CMAKE_On 1 -# define CMAKE_oN 1 -#endif -#if !defined(CMAKE_OFF) -# define CMAKE_OFF 0 -# define CMAKE_OFf 0 -# define CMAKE_OfF 0 -# define CMAKE_Off 0 -# define CMAKE_oFF 0 -# define CMAKE_oFf 0 -# define CMAKE_ofF 0 -# define CMAKE_off 0 -#endif +namespace dindb { + constexpr const int g_current_iface_version = 1; +} //namespace dindb #endif diff --git a/include/backends/db_backend.hpp b/include/backends/db_backend.hpp new file mode 100644 index 0000000..1ef8ce5 --- /dev/null +++ b/include/backends/db_backend.hpp @@ -0,0 +1,91 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id7506CA9825454B80856154ACFE8A9DE2 +#define id7506CA9825454B80856154ACFE8A9DE2 + +#include "backends/backend_loader.hpp" +#include "duckhandy/MaxSizedArray.hpp" +#include +#include +#include +#include +#include +#include + +namespace mchlib { + struct TigerHash; + struct FileRecordData; + struct SetRecordDataFull; +} //namespace mchlib + +namespace dindb { + using GroupIDType = uint32_t; + using FileIDType = uint64_t; + using IDDescMap = std::map; + using ConfirmDeleCallback = std::function; + using TagList = std::vector; + + constexpr const GroupIDType InvalidGroupID = 0; + constexpr const FileIDType InvalidFileID = 0; + + struct LocatedItem { + std::string path; + FileIDType id; + GroupIDType group_id; + }; + + struct LocatedSet { + std::string desc; + GroupIDType id; + uint32_t files_count; + uint32_t dir_count; + }; + + class Backend { + public: + Backend ( void ) = default; + virtual ~Backend ( void ) noexcept = default; + + virtual void connect ( void ) = 0; + virtual void disconnect ( void ) = 0; + + virtual void tag_files ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) = 0; + virtual void tag_files ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) = 0; + virtual void delete_tags ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) = 0; + virtual void delete_tags ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) = 0; + virtual void delete_all_tags ( const std::vector& parFiles, GroupIDType parSet ) = 0; + virtual void delete_all_tags ( const std::vector& parRegexes, GroupIDType parSet ) = 0; + + virtual void delete_group ( const std::vector& parIDs, ConfirmDeleCallback parConf ) = 0; + + virtual void write_files ( const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) = 0; + virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) = 0; + + virtual std::vector locate_in_db ( const std::string& parSearch, const TagList& parTags ) = 0; + virtual std::vector locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) = 0; + virtual std::vector locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive ) = 0; + virtual std::vector locate_sets_in_db ( const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive ) = 0; + + virtual std::vector find_all_sets ( void ) = 0; + virtual std::vector> find_set_details ( const std::vector& parSets ) = 0; + virtual std::vector> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) = 0; + virtual std::vector find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) = 0; + }; +} //namespace dindb + +#endif diff --git a/include/helpers/lengthof.h b/include/backends/exposed_functions.hpp similarity index 58% rename from include/helpers/lengthof.h rename to include/backends/exposed_functions.hpp index 45e78c3..ef174d3 100644 --- a/include/helpers/lengthof.h +++ b/include/backends/exposed_functions.hpp @@ -15,23 +15,20 @@ * along with "dindexer". If not, see . */ -#ifndef id19B690A53A9546D5BD95D89FFF388283 -#define id19B690A53A9546D5BD95D89FFF388283 +#ifndef idA9E47E37E2FA49EE84C2E93FB701C368 +#define idA9E47E37E2FA49EE84C2E93FB701C368 -#if defined(__cplusplus) -# include -#else -# include -#endif +namespace YAML { + class Node; +} //namespace YAML -#if defined(lengthof) -# undef lengthof -#endif -//http://stackoverflow.com/questions/4415524/common-array-length-macro-for-c#4415646 -#if defined(__cplusplus) -# define lengthof(x) ((sizeof(x)/sizeof(0[x])) / ((std::size_t)(!(sizeof(x) % sizeof(0[x]))))) -#else -# define lengthof(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x]))))) -#endif +namespace dindb { + class Backend; +} //namespace dindb + +extern "C" dindb::Backend* dindexer_create_backend ( const YAML::Node* parConfig ); +extern "C" void dindexer_destroy_backend ( dindb::Backend* parDele ); +extern "C" const char* dindexer_backend_name ( void ); +extern "C" int dindexer_backend_iface_version ( void ); #endif diff --git a/include/dindexer-common/settings.hpp b/include/dindexer-common/settings.hpp index e72bb0a..d115c9a 100644 --- a/include/dindexer-common/settings.hpp +++ b/include/dindexer-common/settings.hpp @@ -19,22 +19,16 @@ #define idDC29E3C667BD4793BA0644AE7DC5BD3F #include -#include +#include "backends/backend_loader.hpp" +#include "backends/db_backend.hpp" namespace dinlib { - struct SettingsDB { - std::string address; - std::string username; - std::string password; - std::string dbname; - uint16_t port; - }; - struct Settings { - SettingsDB db; + std::string backend_name; + dindb::BackendPlugin backend_plugin; }; - bool load_settings ( const std::string& parPath, Settings& parOut, bool parExpand=true ); + void load_settings ( const std::string& parPath, Settings& parOut, bool parExpand=true ); } //namespace dinlib #endif diff --git a/include/dindexer-core/searchpaths.hpp b/include/dindexer-core/searchpaths.hpp new file mode 100644 index 0000000..6ab5664 --- /dev/null +++ b/include/dindexer-core/searchpaths.hpp @@ -0,0 +1,66 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idDD90110B9C7E44C4AAF8B46A663B11DC +#define idDD90110B9C7E44C4AAF8B46A663B11DC + +#include +#include +#include +#include +#include + +namespace dincore { + class SearchPaths { + public: + enum SearchType { + Directory, + File, + Any + }; + using Predicate = std::function; + + SearchPaths ( void ) = default; + explicit SearchPaths ( std::vector&& parList ); + SearchPaths ( std::initializer_list parInit ); + ~SearchPaths ( void ) noexcept; + + void add_path ( std::string&& parPath ); + std::string first_hit ( boost::string_ref parFile, SearchType parType=Any ) const; + std::string first_hit ( Predicate parPredicate, SearchType parType=Any ) const; + + private: + std::vector m_paths; + }; + + class ShallowSearchPaths { + public: + ShallowSearchPaths ( void ) = default; + explicit ShallowSearchPaths ( std::vector&& parList ); + ShallowSearchPaths ( std::initializer_list parInit ); + ~ShallowSearchPaths ( void ) noexcept; + + void add_path ( boost::string_ref parPath ); + std::string first_hit ( boost::string_ref parFile, SearchPaths::SearchType parType=SearchPaths::Any ) const; + std::string first_hit ( SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType=SearchPaths::Any ) const; + + private: + std::vector m_paths; + }; +} //namespace dincore + +#endif diff --git a/include/dindexer-common/split_tags.hpp b/include/dindexer-core/split_tags.hpp similarity index 84% rename from include/dindexer-common/split_tags.hpp rename to include/dindexer-core/split_tags.hpp index 149cc25..95a5f6b 100644 --- a/include/dindexer-common/split_tags.hpp +++ b/include/dindexer-core/split_tags.hpp @@ -21,10 +21,11 @@ #include #include #include -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" -namespace dinlib { +namespace dincore { + std::vector split_and_trim ( const std::string& parList, char parSeparator ) a_pure; std::vector split_tags ( const std::string& parCommaSeparatedList ) a_pure; -} //namespace dinlib +} //namespace dincore #endif diff --git a/include/dindexer-machinery/guess_content_type.hpp b/include/dindexer-machinery/guess_content_type.hpp index fb2da6b..e90fcef 100644 --- a/include/dindexer-machinery/guess_content_type.hpp +++ b/include/dindexer-machinery/guess_content_type.hpp @@ -20,7 +20,7 @@ #include "dindexer-machinery/mediatypes.hpp" #include "dindexer-machinery/recorddata.hpp" -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" #include namespace mchlib { diff --git a/include/dindexer-machinery/mediatypes.hpp b/include/dindexer-machinery/mediatypes.hpp index 205179a..a14b078 100644 --- a/include/dindexer-machinery/mediatypes.hpp +++ b/include/dindexer-machinery/mediatypes.hpp @@ -18,7 +18,7 @@ #ifndef id700AFD0F33634ACC88079BB8853A9E13 #define id700AFD0F33634ACC88079BB8853A9E13 -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" #include "enum.h" #include diff --git a/include/dindexer-machinery/recorddata.hpp b/include/dindexer-machinery/recorddata.hpp index 54b1a77..7a13cbb 100644 --- a/include/dindexer-machinery/recorddata.hpp +++ b/include/dindexer-machinery/recorddata.hpp @@ -34,7 +34,7 @@ namespace mchlib { typedef boost::flyweight mime_string; FileRecordData ( void ) = default; - FileRecordData ( const char* parPath, std::time_t parATime, std::time_t parMTime, uint64_t parLevel, bool parIsDir, bool parIsSymLink ) : + FileRecordData ( const char* parPath, std::time_t parATime, std::time_t parMTime, uint16_t parLevel, bool parIsDir, bool parIsSymLink ) : hash {}, abs_path(parPath), mime_full(), @@ -87,9 +87,21 @@ namespace mchlib { bool operator== ( const FileRecordData& parOther ) const; #endif - boost::string_ref path ( void ) const { return boost::string_ref(abs_path).substr(path_offset); } - boost::string_ref mime_type ( void ) const { return boost::string_ref(mime_full.get()).substr(mime_type_offset, mime_type_length); } - boost::string_ref mime_charset ( void ) const { return boost::string_ref(mime_full.get()).substr(mime_charset_offset, mime_charset_length); } + boost::string_ref path ( void ) const { + assert(abs_path.data()); + return boost::string_ref(abs_path).substr(path_offset); + } + + boost::string_ref mime_type ( void ) const { + assert(mime_full.get().data()); + return boost::string_ref(mime_full.get()).substr(mime_type_offset, mime_type_length); + } + + boost::string_ref mime_charset ( void ) const { + assert(mime_full.get().data()); + return boost::string_ref(mime_full.get()).substr(mime_charset_offset, mime_charset_length); + } + void set_mime_parts ( boost::string_ref parType, boost::string_ref parCharset ) { const auto& mime = mime_full.get(); { @@ -130,17 +142,15 @@ namespace mchlib { bool hash_valid; }; - struct SetRecordData { - boost::string_ref name; + struct SetRecordDataFull { + std::string name; + std::string disk_label; + std::string fs_uuid; + uint32_t disk_number; char type; char content_type; }; - struct SetRecordDataFull : public SetRecordData { - std::string name; - uint32_t disk_number; - }; - #if !defined(NDEBUG) inline bool FileRecordData::operator== (const FileRecordData& parOther) const { return (this->abs_path == parOther.abs_path); diff --git a/include/dindexer-machinery/tiger.hpp b/include/dindexer-machinery/tiger.hpp index 8d74daf..b8d540e 100644 --- a/include/dindexer-machinery/tiger.hpp +++ b/include/dindexer-machinery/tiger.hpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace mchlib { struct TigerHash { @@ -35,6 +36,8 @@ namespace mchlib { uint64_t data[3]; uint8_t byte_data[sizeof(uint64_t) * 3]; }; + + bool operator== (const TigerHash& parOther) const { return part_a == parOther.part_a and part_b == parOther.part_b and part_c == parOther.part_c; } }; static_assert(sizeof(TigerHash) == 24, "Wrong struct size"); @@ -42,6 +45,7 @@ namespace mchlib { void tiger_file ( const std::string& parPath, TigerHash& parHashFile, TigerHash& parHashDir, uint64_t& parSizeOut ); void tiger_init_hash ( TigerHash& parHash ); std::string tiger_to_string ( const TigerHash& parHash, bool parUpcase=false ); + TigerHash string_to_tiger ( const std::string& parString ); void tiger_data ( const std::string& parData, TigerHash& parHash ); void tiger_data ( const std::vector& parData, TigerHash& parHash ); } //namespace mchlib diff --git a/include/helpers/compatibility.h b/include/helpers/compatibility.h deleted file mode 100644 index e59a560..0000000 --- a/include/helpers/compatibility.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef id45CDD1DAEF4F42968E3C89F68FDDA9BC -#define id45CDD1DAEF4F42968E3C89F68FDDA9BC - -#if defined(__GNUC__) -# if defined(__clang__) -# if !defined(__has_attribute) - //Fall back to version number comparing -# else -# if __has_attribute(flatten) -# define a_flatten __attribute__((flatten)) -# else -# define a_flatten -# endif -# if __has_attribute(always_inline) -# define a_always_inline __attribute__((always_inline)) -# else -# define a_always_inline -# endif -# if __has_attribute(pure) -# define a_pure __attribute__((pure)) -# else -# define a_pure -# endif -# if __has_attribute(deprecated) -# define a_deprecated __attribute__((deprecated)) -# else -# define a_deprecated -#endif -# endif -# else - //Fix here if you get warnings about unsupported attributes on your compiler -# define a_flatten __attribute__((flatten)) -# define a_always_inline __attribute__((always_inline)) -# define a_pure __attribute__((pure)) -# define a_deprecated __attribute__((deprecated)) -# endif -#else -# warning "Unsupported compiler, please fill this section or file a bug" -# define a_flatten -# define a_always_inline -# define a_pure -# define a_deprecated -#endif - -#endif diff --git a/include/helpers/infix_iterator.hpp b/include/helpers/infix_iterator.hpp deleted file mode 100644 index ca65e34..0000000 --- a/include/helpers/infix_iterator.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// see http://stackoverflow.com/questions/3496982/printing-lists-with-commas-c/3497021#3497021 -// infix_iterator.h -// -// Lifted from Jerry Coffin's 's prefix_ostream_iterator -#if !defined(INFIX_ITERATOR_H_) -#define INFIX_ITERATOR_H_ -#include -#include -template > -class infix_ostream_iterator : - public std::iterator -{ - std::basic_ostream *os; - charT const* delimiter; - bool first_elem; -public: - typedef charT char_type; - typedef traits traits_type; - typedef std::basic_ostream ostream_type; - infix_ostream_iterator(ostream_type& s) - : os(&s),delimiter(0), first_elem(true) - {} - infix_ostream_iterator(ostream_type& s, charT const *d) - : os(&s),delimiter(d), first_elem(true) - {} - infix_ostream_iterator& operator=(T const &item) - { - // Here's the only real change from ostream_iterator: - // Normally, the '*os << item;' would come before the 'if'. - if (!first_elem && delimiter != 0) - *os << delimiter; - *os << item; - first_elem = false; - return *this; - } - infix_ostream_iterator &operator*() { - return *this; - } - infix_ostream_iterator &operator++() { - return *this; - } - infix_ostream_iterator &operator++(int) { - return *this; - } -}; -#endif diff --git a/include/pq/implem/pq_type_helpers.hpp b/include/pq/implem/pq_type_helpers.hpp index 92023be..dc60a6d 100644 --- a/include/pq/implem/pq_type_helpers.hpp +++ b/include/pq/implem/pq_type_helpers.hpp @@ -18,7 +18,7 @@ #ifndef idEC73C3E4D64D44ABA0DB7D41FA8A7EB7 #define idEC73C3E4D64D44ABA0DB7D41FA8A7EB7 -#include "pq/implem/string_bt.hpp" +#include "duckhandy/string_bt.hpp" #include #include #include @@ -30,6 +30,8 @@ struct pg_param; typedef pg_param PGparam; namespace pq { + namespace bt = dhandy::bt; + class Connection; namespace implem { diff --git a/include/pq/implem/sequence_bt.hpp b/include/pq/implem/sequence_bt.hpp deleted file mode 100644 index 80bb7df..0000000 --- a/include/pq/implem/sequence_bt.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef id4FAEF395B9ED47CB9D6B50B54C9A289A -#define id4FAEF395B9ED47CB9D6B50B54C9A289A - -#include - -namespace pq { - namespace bt { - template - struct index_seq { - }; - - namespace implem { - template - struct range_builder; - - template - struct range_builder { - typedef index_seq type; - }; - - template - struct range_builder : public range_builder { - }; - } //namespace implem - - template - using index_range = typename implem::range_builder::type; - } //namespace bt -} //namespace pq - -#endif diff --git a/include/pq/implem/string_bt.hpp b/include/pq/implem/string_bt.hpp deleted file mode 100644 index dc1dbde..0000000 --- a/include/pq/implem/string_bt.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef id170B0E6C34D14EBA9B92A35977BDBFB3 -#define id170B0E6C34D14EBA9B92A35977BDBFB3 - -#include "pq/implem/sequence_bt.hpp" -#include -#include -#include - -namespace pq { - namespace bt { - template - class string; - - template - std::ostream& operator<< ( std::ostream& parStream, const string& parString ); - - template - class string { - public: - friend std::ostream& operator<< <>( std::ostream& parStream, const string& parString ); - constexpr string ( const char* parString ); - - constexpr std::size_t size ( void ) const { return S - 1; } - template - constexpr string operator+ ( const string& parOther ) const; - constexpr char operator[] ( std::size_t parIndex ) const; - - template - constexpr string ( Args... ); - - constexpr const char (&data_arr() const)[S] { return m_data; } - constexpr const char* data() const { return m_data; } - - private: - template - constexpr string ( const index_seq&, const char* parString ); - - const char m_data[S]; - }; - - namespace implem { - template - constexpr string concat ( const index_seq&, const string& parLeft, const string& parRight ) { - return string( - (I < S - 1 ? parLeft[I] : (I < S + S2 - 2 ? parRight[I - (S - 1)] : '\0'))... - ); - } - - } //namespace implem - - template - template - constexpr string::string (const index_seq&, const char* parString) : - m_data{parString[I]...} - { - } - - template - inline constexpr string::string (const char* parString) : - string(index_range<0, S>(), parString) - { - } - - template - template - inline constexpr string::string (Args... parArgs) : - m_data{parArgs...} - { - } - - template - template - constexpr inline string string::operator+ (const string& parOther) const { - return implem::concat(index_range<0, S + S2 - 1>(), string(m_data), parOther); - } - - template - inline std::ostream& operator<< (std::ostream& parStream, const string& parString) { - parStream << parString.m_data; - return parStream; - } - - template - constexpr char string::operator[] (std::size_t parIndex) const { - return (parIndex < S ? m_data[parIndex] : throw std::out_of_range("")); - } - - template - constexpr string make_string (const char (&parData)[S]) { - return string(parData); - } - } //namespace bt -} //namespace pq - -#endif diff --git a/lib/duckhandy b/lib/duckhandy new file mode 160000 index 0000000..9587e0e --- /dev/null +++ b/lib/duckhandy @@ -0,0 +1 @@ +Subproject commit 9587e0e7a19275c462e22b044e9511bac9597605 diff --git a/lib/incredis b/lib/incredis new file mode 160000 index 0000000..94fa688 --- /dev/null +++ b/lib/incredis @@ -0,0 +1 @@ +Subproject commit 94fa688c1784b49689c9a057566bf281016c1634 diff --git a/src/backends/CMakeLists.txt b/src/backends/CMakeLists.txt new file mode 100644 index 0000000..2b73dc9 --- /dev/null +++ b/src/backends/CMakeLists.txt @@ -0,0 +1,37 @@ +project(${bare_name}-backend CXX) + +set(BACKEND_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}") +set(DINDEXER_ENABLED_BACKENDS "postgresql" CACHE STRING "Comma separated list of enabled backends. See directories in backends source directory for a list of known backends.") + +function(ln_backend backend_name) + add_custom_command(TARGET "${backend_name}" POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink $ ${BACKEND_BINARY_DIR}/$ + DEPENDS ${BACKEND_BINARY_DIR}/$ + COMMENT "Creating symlink to ${backend_name}" + ) +endfunction() + +add_library(${PROJECT_NAME} STATIC + backend_loader.cpp +) + +target_include_directories(${PROJECT_NAME} + PUBLIC ${DINDEXER_PUB_INCLUDE_DIR} +) +target_include_directories(${PROJECT_NAME} SYSTEM + PUBLIC ${YAMLCPP_INCLUDE_DIR} +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE ${bare_name}-if + INTERFACE ${YAMLCPP_LIBRARY} + INTERFACE dl + INTERFACE ${bare_name}-machinery +) + +string(REPLACE "," ";" backend_list "${DINDEXER_ENABLED_BACKENDS}") +foreach(backend ${backend_list}) + add_subdirectory(${backend}) + add_dependencies(${PROJECT_NAME} ${bare_name}-backend-${backend}) +endforeach() +unset(backend) diff --git a/src/backends/backend_loader.cpp b/src/backends/backend_loader.cpp new file mode 100644 index 0000000..05b5909 --- /dev/null +++ b/src/backends/backend_loader.cpp @@ -0,0 +1,145 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +//See: +// http://stackoverflow.com/questions/496664/c-dynamic-shared-library-on-linux + +#include "backends/backend_loader.hpp" +#include "backends/exposed_functions.hpp" +#include "backends/backend_version.hpp" +#include +#include +#include +#include + +namespace dindb { + namespace { + BackendPtr load_backend (void* parSOHandle, const YAML::Node* parConfig) { + typedef decltype(&dindexer_create_backend) CreateBackendFun; + typedef decltype(&dindexer_destroy_backend) DeleteBackendFun; + + assert(parSOHandle); + assert(parConfig); + + const char* const fun_name_create = "dindexer_create_backend"; + const char* const fun_name_destroy = "dindexer_destroy_backend"; + + auto create = reinterpret_cast(dlsym(parSOHandle, fun_name_create)); + auto destroy = reinterpret_cast(dlsym(parSOHandle, fun_name_destroy)); + + if (not create) { + std::ostringstream oss; + oss << "Unable to find function " << fun_name_create; + throw std::runtime_error(oss.str()); + } + if (not destroy) { + std::ostringstream oss; + oss << "Unable to find function " << fun_name_destroy; + throw std::runtime_error(oss.str()); + } + return BackendPtr(create(parConfig), destroy); + } + + const char* backend_name (void* parSOHandle) { + typedef decltype(&dindexer_backend_name) GetNameFun; + + assert(parSOHandle); + + auto get_name = reinterpret_cast(dlsym(parSOHandle, "dindexer_backend_name")); + return get_name(); + } + + void nop_destroy (Backend*) { + } + + int backend_iface_version (void* parSOHandle) { + typedef decltype(&dindexer_backend_iface_version) GetVersionFun; + + auto get_version = reinterpret_cast(dlsym(parSOHandle, "dindexer_backend_iface_version")); + return get_version(); + } + } //unnamed namespace + + std::string backend_name (const std::string& parSOPath) { + assert(not parSOPath.empty()); + using SoHandle = std::unique_ptr; + + auto handle = SoHandle(dlopen(parSOPath.c_str(), RTLD_LAZY), &dlclose); + if (handle) + return backend_name(handle.get()); + else + return std::string(); + } + + BackendPlugin::BackendPlugin() : + m_lib(nullptr, &dlclose), + m_backend(nullptr, &nop_destroy), + m_name() + { + } + + BackendPlugin::BackendPlugin (const std::string& parSOPath, const YAML::Node* parConfig) : + m_lib(dlopen(parSOPath.c_str(), RTLD_LAZY), &dlclose), + m_backend(load_backend(m_lib.get(), parConfig)), + m_name(backend_name(m_lib.get())), + m_iface_ver(backend_iface_version(m_lib.get())) + { + if (g_current_iface_version != m_iface_ver) { + m_backend.reset(); + m_lib.reset(); + } + } + + BackendPlugin::~BackendPlugin() noexcept { + } + + const boost::string_ref& BackendPlugin::name() const { + return m_name; + } + + Backend& BackendPlugin::backend() { + assert(m_lib); + assert(m_backend); + if (not m_backend) { + throw std::bad_function_call(); + } + + return *m_backend; + } + + const Backend& BackendPlugin::backend() const { + assert(m_lib); + assert(m_backend); + if (not m_backend) { + throw std::bad_function_call(); + } + + return *m_backend; + } + + bool BackendPlugin::is_loaded() const { + return static_cast(m_backend); + } + + int BackendPlugin::backend_interface_version() const { + return m_iface_ver; + } + + int BackendPlugin::max_supported_interface_version() const { + return g_current_iface_version; + } +} //namespace dindb diff --git a/src/backends/postgresql/CMakeLists.txt b/src/backends/postgresql/CMakeLists.txt new file mode 100644 index 0000000..486d5b0 --- /dev/null +++ b/src/backends/postgresql/CMakeLists.txt @@ -0,0 +1,26 @@ +project(${bare_name}-backend-postgresql CXX) + +add_library(${PROJECT_NAME} SHARED + tag.cpp + delete.cpp + locate.cpp + scan.cpp + navigate.cpp + backend_postgresql.cpp +) + +target_include_directories(${PROJECT_NAME} SYSTEM + PUBLIC ${Boost_INCLUDE_DIRS} +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE ${bare_name}-inc + PRIVATE ${bare_name}-pq +) + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + ARCHIVE DESTINATION lib/static +) +ln_backend(${PROJECT_NAME}) diff --git a/src/backends/postgresql/backend_postgresql.cpp b/src/backends/postgresql/backend_postgresql.cpp new file mode 100644 index 0000000..7dd3d45 --- /dev/null +++ b/src/backends/postgresql/backend_postgresql.cpp @@ -0,0 +1,192 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "backend_postgresql.hpp" +#include "backends/exposed_functions.hpp" +#include "backends/backend_version.hpp" +#include "tag.hpp" +#include "delete.hpp" +#include "scan.hpp" +#include "locate.hpp" +#include "navigate.hpp" +#include "pq/connection.hpp" +#include +#include +#include +#include + +namespace dindb { + namespace { + struct PostgreConnectionSettings { + std::string address; + std::string username; + std::string password; + std::string dbname; + uint16_t port; + }; + } //unnamed namespace +} //namespace dindb + +namespace YAML { + template<> + struct convert { + static Node encode (const dindb::PostgreConnectionSettings& parSettings) { + Node node; + node["address"] = parSettings.address; + node["username"] = parSettings.username; + node["password"] = parSettings.password; + node["port"] = parSettings.port; + node["dbname"] = parSettings.dbname; + return node; + } + + static bool decode (const Node& parNode, dindb::PostgreConnectionSettings& parSettings) { + if (not parNode.IsMap() or parNode.size() != 5) { + return false; + } + + parSettings.address = parNode["address"].as(); + parSettings.username = parNode["username"].as(); + parSettings.password = parNode["password"].as(); + parSettings.dbname = parNode["dbname"].as(); + parSettings.port = parNode["port"].as(); + return true; + } + }; +} //namespace YAML + +namespace dindb { + BackendPostgreSql::BackendPostgreSql (std::string&& parUser, std::string&& parPass, std::string&& parDB, std::string&& parAddr, uint16_t parPort) : + m_conn(new pq::Connection(std::move(parUser), std::move(parPass), std::move(parDB), std::move(parAddr), parPort)) + { + assert(m_conn); + m_conn->connect(); + } + + BackendPostgreSql::~BackendPostgreSql() noexcept { + m_conn->disconnect(); + } + + void BackendPostgreSql::connect() { + m_conn->connect(); + } + + void BackendPostgreSql::disconnect() { + m_conn->disconnect(); + } + + void BackendPostgreSql::tag_files (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + dindb::tag_files(*m_conn, parFiles, parTags, parSet); + } + + void BackendPostgreSql::tag_files (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + dindb::tag_files(*m_conn, parRegexes, parTags, parSet); + } + + void BackendPostgreSql::delete_tags (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + dindb::delete_tags(*m_conn, parFiles, parTags, parSet); + } + + void BackendPostgreSql::delete_tags (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + dindb::delete_tags(*m_conn, parRegexes, parTags, parSet); + } + + void BackendPostgreSql::delete_all_tags (const std::vector& parFiles, GroupIDType parSet) { + dindb::delete_all_tags(*m_conn, parFiles, parSet); + } + + void BackendPostgreSql::delete_all_tags (const std::vector& parRegexes, GroupIDType parSet) { + dindb::delete_all_tags(*m_conn, parRegexes, parSet); + } + + void BackendPostgreSql::delete_group (const std::vector& parIDs, ConfirmDeleCallback parConf) { + dindb::delete_group_from_db(*m_conn, parIDs, parConf); + } + + void BackendPostgreSql::write_files (const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) { + dindb::write_to_db(*m_conn, parData, parSetData, parSignature); + } + + bool BackendPostgreSql::search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) { + return dindb::read_from_db(parItem, parSet, *m_conn, parHash); + } + + std::vector BackendPostgreSql::locate_in_db (const std::string& parSearch, const TagList& parTags) { + return dindb::locate_in_db(*m_conn, parSearch, parTags); + } + + std::vector BackendPostgreSql::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) { + return dindb::locate_in_db(*m_conn, parSearch, parTags); + } + + std::vector BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, bool parCaseInsensitive) { + return dindb::locate_sets_in_db(*m_conn, parSearch, parCaseInsensitive); + } + + std::vector BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive) { + return dindb::locate_sets_in_db(*m_conn, parSearch, parSets, parCaseInsensitive); + } + + std::vector BackendPostgreSql::find_all_sets() { + return dindb::find_all_sets(*m_conn); + } + + std::vector> BackendPostgreSql::find_set_details (const std::vector& parSets) { + return dindb::find_set_details< + dindb::SetDetail_ID, + dindb::SetDetail_Desc, + dindb::SetDetail_CreationDate, + dindb::SetDetail_DiskLabel + >(*m_conn, parSets); + } + + std::vector> BackendPostgreSql::find_file_details (GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) { + return dindb::find_file_details(*m_conn, parSetID, parLevel, parDir); + } + + std::vector BackendPostgreSql::find_paths_starting_by (GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) { + return dindb::find_paths_starting_by(*m_conn, parGroupID, parLevel, parPath); + } +} //namespace dindb + +extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) { + if (not parConfig) + return nullptr; + + auto& config_node = *parConfig; + auto config = config_node["connection"].as(); + return new dindb::BackendPostgreSql( + std::move(config.username), + std::move(config.password), + std::move(config.dbname), + std::move(config.address), + config.port + ); +} + +extern "C" void dindexer_destroy_backend (dindb::Backend* parDele) { + if (parDele) + delete parDele; +} + +extern "C" const char* dindexer_backend_name() { + return "postgresql"; +} + +extern "C" int dindexer_backend_iface_version() { + return dindb::g_current_iface_version; +} diff --git a/src/backends/postgresql/backend_postgresql.hpp b/src/backends/postgresql/backend_postgresql.hpp new file mode 100644 index 0000000..56d7b13 --- /dev/null +++ b/src/backends/postgresql/backend_postgresql.hpp @@ -0,0 +1,67 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idBE5BBAFEC7334F39AC9338F193703341 +#define idBE5BBAFEC7334F39AC9338F193703341 + +#include "backends/db_backend.hpp" +#include +#include +#include + +namespace pq { + class Connection; +} //namespace pq + +namespace dindb { + class BackendPostgreSql : public Backend { + public: + BackendPostgreSql ( BackendPostgreSql&& ) = default; + BackendPostgreSql ( std::string&& parUser, std::string&& parPass, std::string&& parDB, std::string&& parAddr, uint16_t parPort ); + virtual ~BackendPostgreSql ( void ) noexcept; + + virtual void connect ( void ) override; + virtual void disconnect ( void ) override; + + virtual void tag_files ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) override; + virtual void tag_files ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_tags ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_tags ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_all_tags ( const std::vector& parFiles, GroupIDType parSet ) override; + virtual void delete_all_tags ( const std::vector& parRegexes, GroupIDType parSet ) override; + + virtual void delete_group ( const std::vector& parIDs, ConfirmDeleCallback parConf ) override; + + virtual void write_files ( const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) override; + virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) override; + + virtual std::vector locate_in_db ( const std::string& parSearch, const TagList& parTags ) override; + virtual std::vector locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) override; + virtual std::vector locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive ) override; + virtual std::vector locate_sets_in_db ( const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive ) override; + + virtual std::vector find_all_sets ( void ) override; + virtual std::vector> find_set_details ( const std::vector& parSets ) override; + virtual std::vector> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) override; + virtual std::vector find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) override; + + private: + std::unique_ptr m_conn; + }; +} //namespace dindb + +#endif diff --git a/src/delete/postgre_delete.cpp b/src/backends/postgresql/delete.cpp similarity index 72% rename from src/delete/postgre_delete.cpp rename to src/backends/postgresql/delete.cpp index 6fe550c..1304ce8 100644 --- a/src/delete/postgre_delete.cpp +++ b/src/backends/postgresql/delete.cpp @@ -15,10 +15,9 @@ * along with "dindexer". If not, see . */ -#include "postgre_delete.hpp" +#include "delete.hpp" #include "pq/connection.hpp" -#include "dindexer-common/settings.hpp" -#include "helpers/infix_iterator.hpp" +#include "duckhandy/infix_iterator.hpp" #include #include #include @@ -26,10 +25,11 @@ #include #include #include +#include -namespace din { +namespace dindb { namespace { - IDDescMap fetch_existing_ids (pq::Connection& parConn, const std::vector& parIDs) { + IDDescMap fetch_existing_ids (pq::Connection& parConn, const std::vector& parIDs) { using boost::lexical_cast; IDDescMap retmap; @@ -39,7 +39,7 @@ namespace din { std::ostringstream oss; oss << "SELECT \"id\",\"desc\" FROM \"sets\" WHERE \"id\"="; - boost::copy(parIDs, infix_ostream_iterator(oss, " OR \"id\"=")); + boost::copy(parIDs, infix_ostream_iterator(oss, " OR \"id\"=")); oss << ';'; auto resultset = parConn.query(oss.str()); @@ -50,10 +50,10 @@ namespace din { } } //unnamed namespace - void delete_group_from_db (const dinlib::SettingsDB& parDB, const std::vector& parIDs, ConfirmDeleCallback parConf) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); - const auto dele_ids = fetch_existing_ids(conn, parIDs); + void delete_group_from_db (pq::Connection& parDB, const std::vector& parIDs, ConfirmDeleCallback parConf) { + assert(parDB.is_connected()); + + const auto dele_ids = fetch_existing_ids(parDB, parIDs); if (dele_ids.empty()) { return; } @@ -65,11 +65,11 @@ namespace din { ids.reserve(dele_ids.size()); std::ostringstream oss; oss << "BEGIN;\nDELETE FROM \"files\" WHERE \"group_id\"="; - boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator(oss, " OR \"group_id\"=")); + boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator(oss, " OR \"group_id\"=")); oss << ";\nDELETE FROM \"sets\" WHERE \"id\"="; - boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator(oss, " OR \"id\"=")); + boost::copy(dele_ids | boost::adaptors::map_keys, infix_ostream_iterator(oss, " OR \"id\"=")); oss << ";\nCOMMIT;"; - conn.query(oss.str()); + parDB.query(oss.str()); } -} //namespace din +} //namespace dindb diff --git a/src/delete/postgre_delete.hpp b/src/backends/postgresql/delete.hpp similarity index 76% rename from src/delete/postgre_delete.hpp rename to src/backends/postgresql/delete.hpp index f6221b5..befe62a 100644 --- a/src/delete/postgre_delete.hpp +++ b/src/backends/postgresql/delete.hpp @@ -18,21 +18,21 @@ #ifndef idB070B86E0E4047B1AF4144DEF2759F3C #define idB070B86E0E4047B1AF4144DEF2759F3C +#include "backends/db_backend.hpp" #include #include #include -#include #include -namespace dinlib { - struct SettingsDB; -} //namespace dinlib +namespace pq { + class Connection; +} //namespace pq -namespace din { - using IDDescMap = std::map; +namespace dindb { + using IDDescMap = std::map; using ConfirmDeleCallback = std::function; - void delete_group_from_db ( const dinlib::SettingsDB& parDB, const std::vector& parIDs, ConfirmDeleCallback parConf ); -} //namespace din + void delete_group_from_db ( pq::Connection& parDB, const std::vector& parIDs, ConfirmDeleCallback parConf ); +} //namespace dindb #endif diff --git a/src/navigate/flatinsertin2dlist.hpp b/src/backends/postgresql/flatinsertin2dlist.hpp similarity index 74% rename from src/navigate/flatinsertin2dlist.hpp rename to src/backends/postgresql/flatinsertin2dlist.hpp index b8d2a20..fcabeb4 100644 --- a/src/navigate/flatinsertin2dlist.hpp +++ b/src/backends/postgresql/flatinsertin2dlist.hpp @@ -18,13 +18,21 @@ #ifndef idAA901DA47E234E37B325B3192EF50423 #define idAA901DA47E234E37B325B3192EF50423 +//Use this function to turn a 1D sequence of elements that are logically grouped +//together by position into a 2D list. For example, if you have an input sequence +//like {A1, B1, C1, A2, B2, C2, A3, B3, C3} and you want to turn it into +//{ {A1, B1, C1}, {A2, B2, C2}, {A3, B3, C3} } you can use these functions to +//achieve that. You need to specify the size of the innermost groups, 3 in the +//example just given, and push_back() will automatically move to the next +//sublist every 3 items. + #include #include #include #include #include -namespace din { +namespace dhandy { template class FlatInsertIn2DList { public: @@ -44,7 +52,7 @@ namespace din { const std::size_t m_inner_count; const std::size_t m_outer_count; }; -} //namespace din +} //namespace dhandy #include "flatinsertin2dlist.inl" diff --git a/src/navigate/flatinsertin2dlist.inl b/src/backends/postgresql/flatinsertin2dlist.inl similarity index 98% rename from src/navigate/flatinsertin2dlist.inl rename to src/backends/postgresql/flatinsertin2dlist.inl index edb9f09..f03d929 100644 --- a/src/navigate/flatinsertin2dlist.inl +++ b/src/backends/postgresql/flatinsertin2dlist.inl @@ -15,7 +15,7 @@ * along with "dindexer". If not, see . */ -namespace din { +namespace dhandy { template FlatInsertIn2DList::FlatInsertIn2DList (list_type* parList, std::size_t parInnerCount, std::size_t parOuterCount) : m_list(parList), @@ -70,4 +70,4 @@ namespace din { return (m_list->size() - 1) * m_inner_count + m_list[m_list->size() - 1].size(); } } -} //namespace din +} //namespace dhandy diff --git a/src/locate/postgre_locate.cpp b/src/backends/postgresql/locate.cpp similarity index 71% rename from src/locate/postgre_locate.cpp rename to src/backends/postgresql/locate.cpp index 00e0361..259f5c7 100644 --- a/src/locate/postgre_locate.cpp +++ b/src/backends/postgresql/locate.cpp @@ -15,26 +15,19 @@ * along with "dindexer". If not, see . */ -#include "postgre_locate.hpp" +#include "locate.hpp" +#include "dindexer-machinery/recorddata.hpp" #include "pq/connection.hpp" +#include "query_count_limit.hpp" #include "dindexer-machinery/tiger.hpp" #include #include #include +#include -namespace din { +namespace dindb { namespace { - const int g_max_results = 200; - - pq::Connection make_pq_conn ( const dinlib::SettingsDB& parDB, bool parOpen=true ); - - pq::Connection make_pq_conn (const dinlib::SettingsDB& parDB, bool parOpen) { - auto conn = pq::Connection(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - if (parOpen) { - conn.connect(); - } - return conn; - } + const int g_max_results = g_query_count_limit; std::vector sets_result_to_vec (pq::ResultSet&& parResult) { using boost::lexical_cast; @@ -94,21 +87,21 @@ namespace din { } } //unnamed namespace - std::vector locate_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const TagList& parTags) { - auto conn = make_pq_conn(parDB); + std::vector locate_in_db (pq::Connection& parDB, const std::string& parSearch, const TagList& parTags) { + assert(parDB.is_connected()); const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"path\" ~ $1"; - return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch); + return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch); } - std::vector locate_in_db (const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) { - auto conn = make_pq_conn(parDB); + std::vector locate_in_db (pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) { + assert(parDB.is_connected()); const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"hash\"=$1"; - return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true)); + return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true)); } - std::vector locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, bool parCaseInsensitive) { - auto conn = make_pq_conn(parDB); + std::vector locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive) { + assert(parDB.is_connected()); const std::string query = std::string("SELECT \"id\", \"desc\", " "(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", " @@ -116,16 +109,16 @@ namespace din { "FROM \"sets\" WHERE str_match_partial(\"desc\", $1, $2) LIMIT " ) + std::to_string(g_max_results) + ";"; - auto result = conn.query(query, parSearch, parCaseInsensitive); + auto result = parDB.query(query, parSearch, parCaseInsensitive); return sets_result_to_vec(std::move(result)); } - std::vector locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive) { + std::vector locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive) { if (parSets.empty()) { return locate_sets_in_db(parDB, parSearch, parCaseInsensitive); } - auto conn = make_pq_conn(parDB); + assert(parDB.is_connected()); const std::string query = std::string("SELECT \"id\", \"desc\", " "(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", " @@ -133,7 +126,7 @@ namespace din { "FROM \"sets\" WHERE \"id\" = ANY($1) AND str_match_partial(\"desc\", $3, $2) LIMIT " ) + std::to_string(g_max_results) + ";"; - auto result = conn.query(query, parSearch, parCaseInsensitive, parSets); + auto result = parDB.query(query, parSearch, parCaseInsensitive, parSets); return sets_result_to_vec(std::move(result)); } -} //namespace din +} //namespace dindb diff --git a/src/backends/postgresql/locate.hpp b/src/backends/postgresql/locate.hpp new file mode 100644 index 0000000..fd58ed1 --- /dev/null +++ b/src/backends/postgresql/locate.hpp @@ -0,0 +1,37 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id1AE05A59AE0E4A4490040FD85D9AF665 +#define id1AE05A59AE0E4A4490040FD85D9AF665 + +#include +#include +#include +#include "backends/db_backend.hpp" + +namespace pq { + class Connection; +} //namespace pq + +namespace dindb { + std::vector locate_in_db ( pq::Connection& parDB, const std::string& parSearch, const TagList& parTags ); + std::vector locate_in_db ( pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags ); + std::vector locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive ); + std::vector locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive ); +} //namespace dindb + +#endif diff --git a/src/backends/postgresql/navigate.cpp b/src/backends/postgresql/navigate.cpp new file mode 100644 index 0000000..f2f7057 --- /dev/null +++ b/src/backends/postgresql/navigate.cpp @@ -0,0 +1,155 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "navigate.hpp" +#include "pq/connection.hpp" +#include "duckhandy/infix_iterator.hpp" +#include "query_count_limit.hpp" +#include +#include +#include +#include +#include +#include + +namespace dindb { + namespace { + const uint32_t g_files_query_limit = g_query_count_limit; + + std::ostream& operator<< (std::ostream& parOut, const std::vector& parCols) { + parOut << '"'; + boost::copy(parCols, infix_ostream_iterator(parOut, "\", \"")); + parOut << '"'; + return parOut; + } + } //unnamed namespace + + namespace implem { + const SetDetailsMap g_set_details_map { + {SetDetail_Desc, "desc"}, + {SetDetail_Type, "type"}, + {SetDetail_CreationDate, "creation"}, + {SetDetail_AppName, "app_name"}, + {SetDetail_ID, "id"}, + {SetDetail_DiskLabel, "disk_label"}, + {SetDetail_FSUuid, "fs_uuid"} + }; + const FileDetailsMap g_file_details_map { + {FileDetail_ID, "id"}, + {FileDetail_Path, "path"}, + {FileDetail_Level, "level"}, + {FileDetail_GroupID, "group_id"}, + {FileDetail_IsDir, "is_directory"}, + {FileDetail_IsSymLink, "is_symlink"}, + {FileDetail_Size, "size"}, + {FileDetail_Hash, "hash"}, + {FileDetail_IsHashValid, "is_hash_valid"}, + {FileDetail_ATime, "access_time"}, + {FileDetail_MTime, "modify_time"}, + {FileDetail_Unreadable, "unreadable"}, + {FileDetail_MimeType, "mimetype"}, + {FileDetail_Charset, "charset"} + }; + + void query_no_conditions (pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parTable, const std::vector& parIDs, std::function parCallback) { + assert(parDB.is_connected()); + + std::ostringstream oss; + oss << "SELECT " << parCols << ' ' << + "FROM \"" << parTable << "\" " << + "WHERE \"id\"=ANY($1) " << + "ORDER BY \"desc\" ASC " << + "LIMIT " << g_files_query_limit << ';'; + + auto result = parDB.query(oss.str(), parIDs); + for (auto row : result) { + for (auto val : row) { + parCallback(std::move(val)); + } + } + } + + void query_files_in_dir (pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, GroupIDType parGroupID, QueryCallback parCallback) { + assert(parDB.is_connected()); + + std::ostringstream oss; + oss << "SELECT " << parCols << ' ' << + "FROM \"files\" WHERE " << + "\"level\"=$1 " << + "AND \"group_id\"=$2 " << + "AND str_begins_with(\"path\", COALESCE($3, '')) " << + "ORDER BY \"is_directory\" DESC, \"path\" ASC " << + "LIMIT " << g_files_query_limit << ';'; + + auto result = parDB.query( + oss.str(), + parLevel, + parGroupID, + parDir + ); + for (auto row : result) { + for (auto val : row) { + parCallback(std::move(val)); + } + } + } + } //namespace implem + + std::vector find_all_sets (pq::Connection& parDB) { + using boost::lexical_cast; + + assert(parDB.is_connected()); + + const std::string query = "SELECT \"id\" FROM \"sets\";"; + auto res = parDB.query(query); + std::vector retval; + + retval.reserve(res.size()); + for (const auto& row : res) { + retval.push_back(lexical_cast(row[0])); + } + return retval; + } + + std::vector find_paths_starting_by (pq::Connection& parDB, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) { + assert(parDB.is_connected()); + + std::ostringstream oss; + oss << "SELECT \"path\" ||\n" << + "(SELECT CASE \"is_directory\"\n" << + "WHEN TRUE THEN '/'\n" << + "ELSE ''\n" << + "END) as path FROM \"files\" WHERE \"group_id\"=$1 AND " << + "\"level\"=$2 AND str_begins_with(\"path\", COALESCE($3, '')) " << + "ORDER BY \"is_directory\" DESC, \"path\" ASC LIMIT " << + g_files_query_limit << ';'; + + auto result = parDB.query( + oss.str(), + parGroupID, + parLevel, + parPath + ); + std::vector retval; + retval.reserve(retval.size()); + for (auto row : result) { + assert(not row.empty()); + retval.push_back(row[0]); + } + return retval; + } +} //namespace dindb diff --git a/src/navigate/dbsource.hpp b/src/backends/postgresql/navigate.hpp similarity index 60% rename from src/navigate/dbsource.hpp rename to src/backends/postgresql/navigate.hpp index 5da8df9..1a9a8d9 100644 --- a/src/navigate/dbsource.hpp +++ b/src/backends/postgresql/navigate.hpp @@ -20,7 +20,8 @@ #include "dindexer-machinery/recorddata.hpp" #include "flatinsertin2dlist.hpp" -#include "MaxSizedArray.hpp" +#include "duckhandy/MaxSizedArray.hpp" +#include "backends/db_backend.hpp" #include #include #include @@ -31,21 +32,21 @@ #include #include -namespace dinlib { - struct SettingsDB; -} //namespace dinlib - namespace pq { class Connection; } //namespace pq -namespace din { +namespace dindb { + using dhandy::MaxSizedArray; + enum SetDetails { SetDetail_Desc = 0x01, SetDetail_Type = 0x02, - SetDetail_CreeationDate = 0x04, + SetDetail_CreationDate = 0x04, SetDetail_AppName = 0x08, - SetDetail_ID = 0x10 + SetDetail_ID = 0x10, + SetDetail_DiskLabel = 0x20, + SetDetail_FSUuid = 0x40 }; enum FileDetails { @@ -65,40 +66,25 @@ namespace din { FileDetail_Charset = 0x2000 }; - class DBSource { - public: - explicit DBSource ( const dinlib::SettingsDB& parDBSettings ); - ~DBSource ( void ) noexcept; + std::vector find_all_sets ( pq::Connection& parDB ); - void disconnect ( void ); - std::vector sets ( void ); + template + auto find_set_details ( pq::Connection& parDB, const std::vector& parIDs ) -> std::vector>; - template - auto set_details ( const std::vector& parIDs ) -> std::vector>; + template + auto find_file_details ( pq::Connection& parDB, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) -> std::vector>; - template - auto file_details ( uint32_t parSetID, uint16_t parLevel, boost::string_ref parDir ) -> std::vector>; - - std::vector paths_starting_by ( uint32_t parGroupID, uint16_t parLevel, boost::string_ref parPath ); - - private: - struct LocalData; - typedef std::map SetDetailsMap; - typedef std::map FileDetailsMap; - typedef std::vector ColumnList; - typedef std::function QueryCallback; - - pq::Connection& get_conn ( void ); - void query_no_conditions ( const ColumnList& parCols, boost::string_ref parTable, const std::vector& parIDs, QueryCallback parCallback ); - void query_files_in_dir ( const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, uint32_t parGroupID, QueryCallback parCallback ); - - static const SetDetailsMap m_set_details_map; - static const FileDetailsMap m_file_details_map; - - std::unique_ptr m_local_data; - }; + std::vector find_paths_starting_by ( pq::Connection& parDB, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ); namespace implem { + typedef std::vector ColumnList; + typedef std::function QueryCallback; + typedef std::map SetDetailsMap; + typedef std::map FileDetailsMap; + + extern const SetDetailsMap g_set_details_map; + extern const FileDetailsMap g_file_details_map; + template inline std::vector make_columns_vec (const std::map& parDic) { @@ -115,35 +101,42 @@ namespace din { } return columns; } + + void query_no_conditions ( pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parTable, const std::vector& parIDs, QueryCallback parCallback ); + void query_files_in_dir ( pq::Connection& parDB, const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, GroupIDType parGroupID, QueryCallback parCallback ); } //namespace implem template - auto DBSource::set_details (const std::vector& parIDs) -> std::vector> { + inline + auto find_set_details (pq::Connection& parDB, const std::vector& parIDs) -> std::vector> { + using dhandy::FlatInsertIn2DList; typedef std::vector> ReturnType; - typedef void(din::FlatInsertIn2DList::*FlatPushBackFunc)(std::string&&); + typedef void(FlatInsertIn2DList::*FlatPushBackFunc)(std::string&&); - const auto columns = implem::make_columns_vec(m_set_details_map); + const auto columns = implem::make_columns_vec(implem::g_set_details_map); ReturnType list; FlatInsertIn2DList flat_list(&list, sizeof...(D)); FlatPushBackFunc pback_func = &FlatInsertIn2DList::push_back; - this->query_no_conditions(columns, "sets", parIDs, std::bind(pback_func, &flat_list, std::placeholders::_1)); + implem::query_no_conditions(parDB, columns, "sets", parIDs, std::bind(pback_func, &flat_list, std::placeholders::_1)); return list; } template - auto DBSource::file_details (uint32_t parSetID, uint16_t parLevel, boost::string_ref parDir) -> std::vector> { + inline + auto find_file_details (pq::Connection& parDB, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) -> std::vector> { + using dhandy::FlatInsertIn2DList; typedef std::vector> ReturnType; - typedef void(din::FlatInsertIn2DList::*FlatPushBackFunc)(std::string&&); + typedef void(FlatInsertIn2DList::*FlatPushBackFunc)(std::string&&); - const auto columns = implem::make_columns_vec(m_file_details_map); + const auto columns = implem::make_columns_vec(implem::g_file_details_map); ReturnType list; FlatInsertIn2DList flat_list(&list, sizeof...(D)); FlatPushBackFunc pback_func = &FlatInsertIn2DList::push_back; - this->query_files_in_dir(columns, parDir, parLevel, parSetID, std::bind(pback_func, &flat_list, std::placeholders::_1)); + implem::query_files_in_dir(parDB, columns, parDir, parLevel, parSetID, std::bind(pback_func, &flat_list, std::placeholders::_1)); return list; } -} //namespace din +} //namespace dindb #endif diff --git a/src/backends/postgresql/query_count_limit.hpp b/src/backends/postgresql/query_count_limit.hpp new file mode 100644 index 0000000..193fa91 --- /dev/null +++ b/src/backends/postgresql/query_count_limit.hpp @@ -0,0 +1,25 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id367C9335A3444BC0846EF4E2A27AB9A9 +#define id367C9335A3444BC0846EF4E2A27AB9A9 + +namespace dindb { + constexpr const uint32_t g_query_count_limit = 500; +}; + +#endif diff --git a/src/scan/dbbackend.cpp b/src/backends/postgresql/scan.cpp similarity index 77% rename from src/scan/dbbackend.cpp rename to src/backends/postgresql/scan.cpp index eeee541..b1149de 100644 --- a/src/scan/dbbackend.cpp +++ b/src/backends/postgresql/scan.cpp @@ -15,9 +15,8 @@ * along with "dindexer". If not, see . */ -#include "dbbackend.hpp" +#include "scan.hpp" #include "pq/connection.hpp" -#include "dindexer-common/settings.hpp" #include "dindexer-machinery/recorddata.hpp" #include #include @@ -27,20 +26,20 @@ #include #include #include +#include -namespace din { +namespace dindb { namespace { } //unnamed namespace - bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parHash) { + bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash) { using boost::lexical_cast; - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + assert(parDB.is_connected()); uint32_t group_id; { - auto resultset = conn.query( + auto resultset = parDB.query( "SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash=$1 LIMIT 1;", tiger_to_string(parHash, true) ); @@ -64,8 +63,8 @@ namespace din { } { - auto resultset = conn.query( - "SELECT \"desc\",\"type\",\"disk_number\",\"content_type\" FROM sets WHERE \"id\"=$1;", + auto resultset = parDB.query( + "SELECT \"desc\",\"type\",\"disk_number\",\"fs_uuid\",\"disk_label\",\"content_type\" FROM sets WHERE \"id\"=$1;", group_id ); if (resultset.empty()) { @@ -78,33 +77,36 @@ namespace din { parSet.type = lexical_cast(row["type"]); parSet.name = row["desc"]; parSet.disk_number = lexical_cast(row["disk_number"]); + parSet.fs_uuid = row["fs_uuid"]; + parSet.disk_label = row["disk_label"]; parSet.content_type = lexical_cast(row["content_type"]); } return true; } - void write_to_db (const dinlib::SettingsDB& parDB, const std::vector& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) { + void write_to_db (pq::Connection& parDB, const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) { using std::chrono::system_clock; using boost::lexical_cast; + assert(parDB.is_connected()); + if (parData.empty()) { return; } - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); - - conn.query("BEGIN;"); + parDB.query("BEGIN;"); uint32_t new_group_id; { - auto id_res = conn.query("INSERT INTO \"sets\" " + auto id_res = parDB.query("INSERT INTO \"sets\" " "(\"desc\",\"type\",\"app_name\"" - ",\"content_type\") " - "VALUES ($1, $2, $3, $4) RETURNING \"id\";", + ",\"content_type\",\"fs_uuid\",\"disk_label\") " + "VALUES ($1, $2, $3, $4, $5, $6) RETURNING \"id\";", parSetData.name, boost::string_ref(&parSetData.type, 1), parSignature, - boost::string_ref(&parSetData.content_type, 1) + boost::string_ref(&parSetData.content_type, 1), + parSetData.fs_uuid, + parSetData.disk_label ); assert(id_res.size() == 1); assert(id_res[0].size() == 1); @@ -127,7 +129,7 @@ namespace din { const auto& itm = parData[z]; assert(itm.path().data()); - conn.query(query, + parDB.query(query, (itm.path().empty() ? empty_path_string : itm.path()), tiger_to_string(itm.hash), itm.level, @@ -143,6 +145,6 @@ namespace din { itm.mime_charset() ); } - conn.query("COMMIT;"); + parDB.query("COMMIT;"); } -} //namespace din +} //namespace dindb diff --git a/src/scan/dbbackend.hpp b/src/backends/postgresql/scan.hpp similarity index 72% rename from src/scan/dbbackend.hpp rename to src/backends/postgresql/scan.hpp index 943c74e..b270c1a 100644 --- a/src/scan/dbbackend.hpp +++ b/src/backends/postgresql/scan.hpp @@ -22,20 +22,21 @@ #include #include -namespace dinlib { - struct SettingsDB;; -} //namespace dinlib - namespace mchlib { struct FileRecordData; - struct SetRecordData; struct SetRecordDataFull; struct TigerHash; } //namespace mchlib -namespace din { - void write_to_db ( const dinlib::SettingsDB& parDB, const std::vector& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature ); - bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parHash ); -} //namespace din +namespace pq { + class Connection; +} //namespace pq + +namespace dindb { + struct Settings;; + + void write_to_db ( pq::Connection& parDB, const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ); + bool read_from_db ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, pq::Connection& parDB, const mchlib::TigerHash& parHash ); +} //namespace dindb #endif diff --git a/src/tag/tag_postgres.cpp b/src/backends/postgresql/tag.cpp similarity index 56% rename from src/tag/tag_postgres.cpp rename to src/backends/postgresql/tag.cpp index 7a41096..7f0e999 100644 --- a/src/tag/tag_postgres.cpp +++ b/src/backends/postgresql/tag.cpp @@ -15,141 +15,135 @@ * along with "dindexer". If not, see . */ -#include "tag_postgres.hpp" +#include "tag.hpp" #include "pq/connection.hpp" -#include "dindexer-common/settings.hpp" #include +#include -namespace din { - void tag_files (const dinlib::SettingsDB& parDB, const std::vector& parFiles, const std::vector& parTags, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); +namespace dindb { + void tag_files (pq::Connection& parDB, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + assert(parDB.is_connected()); - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"id\"=ANY($2) AND \"group_id\"=$3;"; - conn.query(query, parTags, parFiles, parSet.group_id); + parDB.query(query, parTags, parFiles, parSet); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"id\"=ANY($2);"; - conn.query(query, parTags, parFiles); + parDB.query(query, parTags, parFiles); } } - void tag_files (const dinlib::SettingsDB& parDB, const std::vector& parRegexes, const std::vector& parTags, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + void tag_files (pq::Connection& parDB, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + assert(parDB.is_connected()); - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { if (parRegexes.size() == 1) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"group_id\"=$2 AND \"path\" ~ $3;"; - conn.query(query, parTags, parSet.group_id, parRegexes.front()); + parDB.query(query, parTags, parSet, parRegexes.front()); } else if (parRegexes.size() > 1) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"group_id\"=$2 AND \"path\" ~ ANY($3);"; - conn.query(query, parTags, parSet.group_id, parRegexes); + parDB.query(query, parTags, parSet, parRegexes); } else if (parRegexes.size() == 0) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) WHERE \"group_id\"=$2 ORDER BY 1);"; - conn.query(query, parTags, parSet.group_id); + parDB.query(query, parTags, parSet); } } else { if (parRegexes.size() == 1) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"path\" ~ $2;"; - conn.query(query, parTags, parRegexes.front()); + parDB.query(query, parTags, parRegexes.front()); } else if (parRegexes.size() > 1) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1) WHERE \"path\" ~ ANY($2);"; - conn.query(query, parTags, parRegexes); + parDB.query(query, parTags, parRegexes); } else if (parRegexes.size() == 0) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT DISTINCT UNNEST(\"tags\" || $1) ORDER BY 1);"; - conn.query(query, parTags); + parDB.query(query, parTags); } } } - void delete_tags (const dinlib::SettingsDB& parDB, const std::vector& parFiles, const std::vector& parTags, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + void delete_tags (pq::Connection& parDB, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + assert(parDB.is_connected()); if (parTags.size() == 1) { - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"id\" = ANY($2) AND \"group_id\" = $3;"; - conn.query(query, parTags.front(), parFiles, parSet.group_id); + parDB.query(query, parTags.front(), parFiles, parSet); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"id\" = ANY($2);"; - conn.query(query, parTags.front(), parFiles); + parDB.query(query, parTags.front(), parFiles); } } else { - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"id\" = ANY($2) AND \"group_id\" = $3;"; - conn.query(query, parTags, parFiles, parSet.group_id); + parDB.query(query, parTags, parFiles, parSet); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"id\" = ANY($2);"; - conn.query(query, parTags, parFiles); + parDB.query(query, parTags, parFiles); } } } - void delete_tags (const dinlib::SettingsDB& parDB, const std::vector& parRegexes, const std::vector& parTags, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + void delete_tags (pq::Connection& parDB, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + assert(parDB.is_connected()); if (parTags.size() == 1) { - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"group_id\" = $3 AND \"path\" ~ ANY($3);"; - conn.query(query, parTags.front(), parSet.group_id, parRegexes); + parDB.query(query, parTags.front(), parSet, parRegexes); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY_REMOVE(tags, $1) WHERE \"path\" ~ ANY($2);"; - conn.query(query, parTags.front(), parRegexes); + parDB.query(query, parTags.front(), parRegexes); } } else { - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"group_id\" = $2 AND \"path\" ~ ANY($3);"; - conn.query(query, parTags, parSet.group_id, parRegexes); + parDB.query(query, parTags, parSet, parRegexes); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = ARRAY(SELECT UNNEST(\"tags\") EXCEPT SELECT UNNEST($1)) WHERE \"path\" = ANY($2);"; - conn.query(query, parTags, parRegexes); + parDB.query(query, parTags, parRegexes); } } } - void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector& parFiles, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + void delete_all_tags (pq::Connection& parDB, const std::vector& parFiles, GroupIDType parSet) { + assert(parDB.is_connected()); - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"id\"=ANY($1) AND \"group_id\"=$2;"; - conn.query(query, parFiles, parSet.group_id); + parDB.query(query, parFiles, parSet); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"id\"=ANY($1);"; - conn.query(query, parFiles); + parDB.query(query, parFiles); } } - void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector& parRegexes, OwnerSetInfo parSet) { - pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port); - conn.connect(); + void delete_all_tags (pq::Connection& parDB, const std::vector& parRegexes, GroupIDType parSet) { + assert(parDB.is_connected()); - if (parSet.is_valid) { + if (InvalidGroupID != parSet) { const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"group_id\"=$1 AND \"path\" ~ ANY($2);"; - conn.query(query, parSet.group_id, parRegexes); + parDB.query(query, parSet, parRegexes); } else { const std::string query = "UPDATE \"files\" SET \"tags\" = '{}' WHERE \"path\" ~ ANY($2);"; - conn.query(query, parRegexes); + parDB.query(query, parRegexes); } } -} //namespace din +} //namespace dindb diff --git a/src/backends/postgresql/tag.hpp b/src/backends/postgresql/tag.hpp new file mode 100644 index 0000000..ed5b7e4 --- /dev/null +++ b/src/backends/postgresql/tag.hpp @@ -0,0 +1,43 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idE1E1650A8CAA4949BD6D4D58BF2599F5 +#define idE1E1650A8CAA4949BD6D4D58BF2599F5 + +#include "backends/db_backend.hpp" +#include +#include +#include +#include + +namespace pq { + class Connection; +} //namespace pq + +namespace dindb { + struct Settings; + + void tag_files ( pq::Connection& parDB, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ); + void tag_files ( pq::Connection& parDB, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ); + + void delete_tags ( pq::Connection& parDB, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ); + void delete_tags ( pq::Connection& parDB, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ); + void delete_all_tags ( pq::Connection& parDB, const std::vector& parFiles, GroupIDType parSet ); + void delete_all_tags ( pq::Connection& parDB, const std::vector& parRegexes, GroupIDType parSet ); +} //namespace dindb + +#endif diff --git a/src/backends/redis/CMakeLists.txt b/src/backends/redis/CMakeLists.txt new file mode 100644 index 0000000..538504a --- /dev/null +++ b/src/backends/redis/CMakeLists.txt @@ -0,0 +1,56 @@ +project(${bare_name}-backend-redis CXX) + +set(DINDEXER_REDIS_SCRIPTS_PATH "${CMAKE_INSTALL_PREFIX}/${bare_name}/redis" CACHE STRING "Path where Lua scripts for Redis are stored") + +find_package(Boost 1.53.0 REQUIRED COMPONENTS regex) + +add_library(${PROJECT_NAME} SHARED + backend_redis.cpp + tag.cpp + delete.cpp + find.cpp +) + +target_include_directories(${PROJECT_NAME} SYSTEM + PUBLIC ${Boost_INCLUDE_DIRS} +) +target_include_directories(${PROJECT_NAME} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} + PUBLIC ${CMAKE_SOURCE_DIR}/src/incredis +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE ${bare_name}-inc + PUBLIC ${bare_name}-core + PRIVATE ${Boost_LIBRARIES} + PRIVATE incredis +) + +configure_file( + redisConfig.h.in + ${CMAKE_CURRENT_BINARY_DIR}/redisConfig.h +) +set(LUA_SCRIPTS + tag_if_in_set.lua + dele_tag_if_in_set.lua + dele_hash.lua +) + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + ARCHIVE DESTINATION lib/static +) + +set(lua_script_list "") +foreach (lua_script ${LUA_SCRIPTS}) + get_filename_component(lua_script_basename "${lua_script}" NAME) + configure_file("${lua_script}" "${CMAKE_CURRENT_BINARY_DIR}/lua/${lua_script_basename}" COPYONLY) + list(APPEND lua_script_list "${CMAKE_CURRENT_BINARY_DIR}/lua/${lua_script_basename}") +endforeach() +unset(lua_script) +unset(lua_script_basename) +install(FILES ${lua_script_list} DESTINATION "${DINDEXER_REDIS_SCRIPTS_PATH}") +unset(lua_script_list) + +ln_backend(${PROJECT_NAME}) diff --git a/src/backends/redis/backend_redis.cpp b/src/backends/redis/backend_redis.cpp new file mode 100644 index 0000000..ec3383a --- /dev/null +++ b/src/backends/redis/backend_redis.cpp @@ -0,0 +1,322 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "backend_redis.hpp" +#include "dindexer-machinery/recorddata.hpp" +#include "backends/exposed_functions.hpp" +#include "backends/backend_version.hpp" +#include "duckhandy/lexical_cast.hpp" +#include "dindexerConfig.h" +#include "redisConfig.h" +#include "duckhandy/stringize.h" +#include "tag.hpp" +#include "delete.hpp" +#include "find.hpp" +#include "record_data_adapt.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace dindb { + namespace { + struct RedisConnectionSettings { + std::string address; + uint16_t port; + uint16_t database; + }; + + std::string read_script (const dincore::SearchPaths& parSearch, const char* parName) { + const auto full_path = parSearch.first_hit(boost::string_ref(parName)); + if (full_path.empty()) { + const std::string msg = std::string("Unable to locate and load Lua script \"") + parName + "\" from any of the given search paths"; + throw std::runtime_error(msg); + } + + std::ifstream script(full_path); + std::string retval; + script.seekg(0, std::ios::end); + retval.reserve(script.tellg()); + script.seekg(0, std::ios::beg); + retval.assign(std::istreambuf_iterator(script), std::istreambuf_iterator()); + + return retval; + } + } //unnamed namespace +} //namespace dindb + +namespace YAML { + template<> + struct convert { + static Node encode (const dindb::RedisConnectionSettings& parSettings) { + Node node; + node["address"] = parSettings.address; + node["port"] = parSettings.port; + node["database"] = parSettings.database; + return node; + } + + static bool decode (const Node& parNode, dindb::RedisConnectionSettings& parSettings) { + if (not parNode.IsMap() or parNode.size() != 2) { + return false; + } + + parSettings.address = parNode["address"].as(); + parSettings.port = parNode["port"].as(); + if (parNode["database"]) + parSettings.database = parNode["database"].as(); + else + parSettings.database = 0; + return true; + } + }; +} //namespace YAML + +namespace dindb { + BackendRedis::BackendRedis(std::string&& parAddress, uint16_t parPort, uint16_t parDatabase, bool parConnect, dincore::SearchPaths&& parLuaPaths) : + m_redis(std::move(parAddress), parPort), + m_tag_if_in_set(), + m_lua_script_paths(std::move(parLuaPaths)), + m_database(parDatabase) + { + if (parConnect) + this->connect(); + } + + BackendRedis::~BackendRedis() noexcept { + } + + void BackendRedis::connect() { + using dhandy::lexical_cast; + + m_redis.connect(); + m_redis.wait_for_connect(); + if (m_redis.is_connected()) { + auto batch = m_redis.make_batch(); + batch.select(m_database); + batch.client_setname(PROGRAM_NAME "_v" STRINGIZE(VERSION_MAJOR) "." STRINGIZE(VERSION_MINOR) "." STRINGIZE(VERSION_PATCH)); + batch.script_flush(); + batch.throw_if_failed(); + } + else { + std::ostringstream oss; + oss << "Error connecting to Redis: " << m_redis.command().connection_error(); + throw std::runtime_error(oss.str()); + } + + m_tag_if_in_set = m_redis.command().make_script(read_script(m_lua_script_paths, "tag_if_in_set.lua")); + m_dele_tag_if_in_set = m_redis.command().make_script(read_script(m_lua_script_paths, "dele_tag_if_in_set.lua")); + m_dele_hash = m_redis.command().make_script(read_script(m_lua_script_paths, "dele_hash.lua")); + } + + void BackendRedis::disconnect() { + m_redis.disconnect(); + } + + void BackendRedis::tag_files (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + dindb::tag_files(m_redis, m_tag_if_in_set, parFiles, parTags, parSet); + } + + void BackendRedis::tag_files (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + dindb::tag_files(m_redis, m_tag_if_in_set, parRegexes, parTags, parSet); + } + + void BackendRedis::delete_tags (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parFiles, parTags, parSet); + } + + void BackendRedis::delete_tags (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parTags, parSet); + } + + void BackendRedis::delete_all_tags (const std::vector& parFiles, GroupIDType parSet) { + dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parFiles, parSet); + } + + void BackendRedis::delete_all_tags (const std::vector& parRegexes, GroupIDType parSet) { + dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parSet); + } + + void BackendRedis::delete_group (const std::vector& parIDs, ConfirmDeleCallback parConf) { + delete_group_from_db(m_redis, m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf); + } + + void BackendRedis::write_files (const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) { + using dhandy::lexical_cast; + using boost::string_ref; + + const auto data_size = static_cast(parData.size()); + const auto group_id_int = m_redis.hincrby(PROGRAM_NAME ":indices", "set", 1); + const auto file_id_int = m_redis.hincrby(PROGRAM_NAME ":indices", "files", data_size); + + const auto group_id = lexical_cast(group_id_int); + const std::string set_key = PROGRAM_NAME ":set:" + group_id; + const std::string level_key = PROGRAM_NAME ":levels:" + group_id; + assert(file_id_int >= data_size); + const auto base_file_id = file_id_int - data_size + 1; + + auto batch = m_redis.make_batch(); + + batch.hmset( + set_key, + "name", parSetData.name, + "disk_label", parSetData.disk_label, + "fs_uuid", parSetData.fs_uuid, + "type", parSetData.type, + "content_type", parSetData.content_type, + "base_file_id", lexical_cast(base_file_id), + "item_count", lexical_cast(parData.size()), + "dir_count", lexical_cast(std::count_if(parData.begin(), parData.end(), [](const mchlib::FileRecordData& r){return r.is_directory;})), + "creation", lexical_cast(std::time(nullptr)), + "app_name", parSignature + ); + +#if !defined(NDEBUG) + std::size_t inserted_count = 0; +#endif + for (auto z = base_file_id; z < base_file_id + data_size; ++z) { + const std::string file_key = PROGRAM_NAME ":file:" + lexical_cast(z); + assert(z >= base_file_id); + assert(static_cast(z - base_file_id) < parData.size()); + const auto& file_data = parData[z - base_file_id]; + const std::string hash = tiger_to_string(file_data.hash); + batch.hmset( + file_key, + "hash", hash, + "path", file_data.path(), + "size", lexical_cast(file_data.size), + "level", lexical_cast(file_data.level), + "mime_type", file_data.mime_type(), + "mime_charset", file_data.mime_charset(), + "is_directory", (file_data.is_directory ? '1' : '0'), + "is_symlink", (file_data.is_symlink ? '1' : '0'), + "unreadable", (file_data.unreadable ? '1' : '0'), + "hash_valid", (file_data.hash_valid ? '1' : '0'), + "group_id", group_id, + "atime", lexical_cast(file_data.atime), + "mtime", lexical_cast(file_data.mtime) + ); + + batch.sadd( + PROGRAM_NAME ":hash:" + hash, + lexical_cast(z) + ); + + batch.zadd(level_key, redis::IncRedisBatch::ZADD_None, false, static_cast(file_data.level), file_key); +#if !defined(NDEBUG) + ++inserted_count; +#endif + } + assert(inserted_count == parData.size()); + + batch.throw_if_failed(); + } + + bool BackendRedis::search_file_by_hash (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) { + using boost::empty; + + const std::string hash_key = PROGRAM_NAME ":hash:" + tiger_to_string(parHash); + auto hash_reply = m_redis.srandmember(hash_key); + if (not hash_reply) { + return false; + } + else { + const auto file_key = PROGRAM_NAME ":file:" + *hash_reply; + auto set_key_and_file_item = redis::range_as(m_redis.hscan(file_key)); + parItem = std::move(set_key_and_file_item.second); + assert(parItem.hash == parHash); + const std::string group_key = PROGRAM_NAME ":set:" + set_key_and_file_item.first; + + auto scan_range = m_redis.hscan(group_key); + if (empty(scan_range)) { + return false; + } + else { + parSet = redis::range_as(m_redis.hscan(group_key)); + return true; + } + } + } + + std::vector BackendRedis::locate_in_db (const std::string& parSearch, const TagList& parTags) { + return dindb::locate_in_db(m_redis, parSearch, parTags); + } + + std::vector BackendRedis::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) { + return dindb::locate_in_db(m_redis, parSearch, parTags); + } + + std::vector BackendRedis::locate_sets_in_db (const std::string& parSubstr, bool parCaseInsensitive) { + return dindb::locate_sets_in_db(m_redis, parSubstr, parCaseInsensitive); + } + + std::vector BackendRedis::locate_sets_in_db (const std::string& parSubstr, const std::vector& parSets, bool parCaseInsensitive) { + return dindb::locate_sets_in_db(m_redis, parSubstr, parSets, parCaseInsensitive); + } + + std::vector BackendRedis::find_all_sets() { + return dindb::find_all_sets(m_redis); + } + + std::vector> BackendRedis::find_set_details (const std::vector& parSets) { + return dindb::find_set_details(m_redis, parSets); + } + + std::vector> BackendRedis::find_file_details (GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) { + return dindb::find_file_details(m_redis, parSetID, parLevel, parDir); + } + + std::vector BackendRedis::find_paths_starting_by (GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) { + return dindb::find_paths_starting_by(m_redis, parGroupID, parLevel, parPath); + } +} //namespace dindb + +extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) { + if (not parConfig) + return nullptr; + + auto& config_node = *parConfig; + auto config = config_node["connection"].as(); + + auto vec = (config_node["script_paths"] ? config_node["script_paths"].as>() : std::vector()); + dincore::SearchPaths lua_paths(std::move(vec)); + lua_paths.add_path(REDIS_SCRIPTS_PATH); + + return new dindb::BackendRedis( + std::move(config.address), + config.port, + config.database, + true, + std::move(lua_paths) + ); +} + +extern "C" void dindexer_destroy_backend (dindb::Backend* parDele) { + if (parDele) + delete parDele; +} + +extern "C" const char* dindexer_backend_name() { + return "redis"; +} + +extern "C" int dindexer_backend_iface_version() { + return dindb::g_current_iface_version; +} diff --git a/src/backends/redis/backend_redis.hpp b/src/backends/redis/backend_redis.hpp new file mode 100644 index 0000000..7e13626 --- /dev/null +++ b/src/backends/redis/backend_redis.hpp @@ -0,0 +1,70 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idB2F92EE07A004D5293FD0657EEE8F75B +#define idB2F92EE07A004D5293FD0657EEE8F75B + +#include "backends/db_backend.hpp" +#include "incredis/incredis.hpp" +#include "incredis/script.hpp" +#include "dindexer-core/searchpaths.hpp" +#include +#include + +namespace dindb { + class BackendRedis : public Backend { + public: + BackendRedis ( BackendRedis&& ) = default; + BackendRedis ( std::string&& parAddress, uint16_t parPort, uint16_t parDatabase, bool parConnect, dincore::SearchPaths&& parLuaPaths ); + virtual ~BackendRedis ( void ) noexcept; + + virtual void connect ( void ) override; + virtual void disconnect ( void ) override; + + virtual void tag_files ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) override; + virtual void tag_files ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_tags ( const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_tags ( const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet ) override; + virtual void delete_all_tags ( const std::vector& parFiles, GroupIDType parSet ) override; + virtual void delete_all_tags ( const std::vector& parRegexes, GroupIDType parSet ) override; + + virtual void delete_group ( const std::vector& parIDs, ConfirmDeleCallback parConf ) override; + + virtual void write_files ( const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature ) override; + virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) override; + + virtual std::vector locate_in_db ( const std::string& parSearch, const TagList& parTags ) override; + virtual std::vector locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) override; + virtual std::vector locate_sets_in_db ( const std::string& parSubstr, bool parCaseInsensitive ) override; + virtual std::vector locate_sets_in_db ( const std::string& parSubstr, const std::vector& parSets, bool parCaseInsensitive ) override; + + virtual std::vector find_all_sets ( void ) override; + virtual std::vector> find_set_details ( const std::vector& parSets ) override; + virtual std::vector> find_file_details ( GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ) override; + virtual std::vector find_paths_starting_by ( GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ) override; + + private: + redis::IncRedis m_redis; + redis::Script m_tag_if_in_set; + redis::Script m_dele_tag_if_in_set; + redis::Script m_dele_hash; + dincore::SearchPaths m_lua_script_paths; + uint16_t m_database; + }; +} //namespace dindb + +#endif diff --git a/src/backends/redis/dele_hash.lua b/src/backends/redis/dele_hash.lua new file mode 100644 index 0000000..078482e --- /dev/null +++ b/src/backends/redis/dele_hash.lua @@ -0,0 +1,27 @@ +-- Copyright 2015, 2016, Michele Santullo +-- This file is part of "dindexer". +-- +-- "dindexer" is free software: you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- "dindexer" is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with "dindexer". If not, see . + +local hash_key = KEYS[1] +local base_index = ARGV[1] +local file_count = ARGV[2] + +for z = base_index, base_index + file_count - 1 do + redis.call("SREM", hash_key, z) +end +if redis.call("SCARD", hash_key) == 0 then + redis.call("DEL", hash_key) +end +return nil diff --git a/src/backends/redis/dele_tag_if_in_set.lua b/src/backends/redis/dele_tag_if_in_set.lua new file mode 100644 index 0000000..eb5ada7 --- /dev/null +++ b/src/backends/redis/dele_tag_if_in_set.lua @@ -0,0 +1,68 @@ +-- Copyright 2015, 2016, Michele Santullo +-- This file is part of "dindexer". +-- +-- "dindexer" is free software: you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- "dindexer" is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with "dindexer". If not, see . + +local tag_key = KEYS[1] +local file_key = KEYS[2] +local group_id = ARGV[1] + +local function split_string(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t = {} + local z = 1 + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + t[z] = str + z = z + 1 + end + return t +end + +local function dele_tag_from_list(tag_list, dele_tag) + tag_list = split_string(tag_list or "", ",") + + local z = 1 + for _,str in ipairs(tag_list) do + if str == dele_tag then + table.remove(tag_list, z) + break + end + z = z + 1 + end + return table.concat(tag_list, ",") +end + +if group_id ~= 0 then + local found_group_id = redis.call("HGET", file_key, "group_id") + if found_group_id ~= group_id then + return nil + end +end + +local dele_tag = split_string(tag_key, ":") +local new_tag_list = dele_tag_from_list(redis.call("HGET", file_key, "tags"), dele_tag[#dele_tag]) + +if new_tag_list == "" then + redis.call("HDEL", file_key, "tags") +else + redis.call("HSET", file_key, "tags", new_tag_list) +end + +local retval = redis.call("SREM", tag_key, file_key) +if redis.call("SCARD", tag_key) == 0 then + redis.call("DEL", tag_key) +end +return retval diff --git a/src/backends/redis/delete.cpp b/src/backends/redis/delete.cpp new file mode 100644 index 0000000..7eab682 --- /dev/null +++ b/src/backends/redis/delete.cpp @@ -0,0 +1,153 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "delete.hpp" +#include "tag.hpp" +#include "incredis/incredis.hpp" +#include "duckhandy/lexical_cast.hpp" +#include "duckhandy/sequence_bt.hpp" +#include "dindexerConfig.h" +#include +#include +#include +#include +#include + +namespace dindb { + namespace { + std::pair confirm_dele (redis::IncRedisBatch& parBatch, const std::vector& parIDs, ConfirmDeleCallback parConf) { + using dhandy::lexical_cast; + + if (parIDs.empty()) + return std::make_pair(false, parIDs.size()); + + for (auto id : parIDs) { + const auto set_key = PROGRAM_NAME ":set:" + lexical_cast(id); + parBatch.hmget(set_key, "base_file_id", "item_count", "name"); + } + + std::map set_dele_list; + std::size_t id_index = 0; + for (const auto& reply : parBatch.replies()) { + const auto res = redis::get_array(reply); + assert(res.size() == 3); + if (redis::RedisVariantType_Nil != res[0].which()) { + assert(redis::RedisVariantType_Nil != res[1].which()); + set_dele_list[parIDs[id_index]] = redis::get_string(res[2]); + } + ++id_index; + } + + if (set_dele_list.empty()) + return std::make_pair(false, set_dele_list.size()); + else + return std::make_pair(parConf(set_dele_list), set_dele_list.size()); + //return std::make_pair(true, set_dele_list.size()); + } + + template + void chunked_run_pvt (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey, dhandy::bt::number_seq) { + for (IT i = 0; i < parCount / CHUNK; ++i) { + parBatch.run(parCommand, parMakeKey(i * CHUNK + parFrom + SVALS)...); + } + for (auto i = ((parFrom + parCount) / CHUNK) * CHUNK; i < parFrom + parCount; ++i) { + const auto key = parMakeKey(i); + parBatch.run(parCommand, key); + } + } + + template + void chunked_run (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey) { + chunked_run_pvt(parBatch, parCommand, parFrom, parCount, parMakeKey, dhandy::bt::number_range()); + }; + } //unnamed namespace + + void delete_group_from_db (redis::IncRedis& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector& parIDs, ConfirmDeleCallback parConf) { + using dhandy::lexical_cast; + using IDRange = std::tuple; + + auto set_batch = parRedis.make_batch(); + + auto dele_pair = confirm_dele(set_batch, parIDs, parConf); + assert(set_batch.batch().replies_requested()); + if (not dele_pair.first) + return; + + std::vector ranges; + ranges.reserve(dele_pair.second); + std::size_t id_index = 0; + for (const auto& reply : set_batch.replies()) { + const auto res = redis::get_array(reply); + if (redis::RedisVariantType_Nil != res[0].which()) { + ranges.push_back( + IDRange( + parIDs[id_index], + lexical_cast(redis::get_string(res[0])), + lexical_cast(redis::get_string(res[1])) + ) + ); + } + ++id_index; + } + assert(ranges.size() == dele_pair.second); + + for (const auto& dele_tuple : ranges) { + const auto set_id = std::get<0>(dele_tuple); + const auto file_base_index = std::get<1>(dele_tuple); + const auto file_count = std::get<2>(dele_tuple); + + std::vector ids; + ids.reserve(file_count); + std::copy( + boost::counting_iterator(file_base_index), + boost::counting_iterator(file_base_index + file_count), + std::back_inserter(ids) + ); + + delete_all_tags(parRedis, parDeleTagIfInSet, ids, set_id); + } + + auto dele_batch = parRedis.make_batch(); + for (const auto& dele_tuple : ranges) { + const auto set_id = std::get<0>(dele_tuple); + const auto file_base_index = std::get<1>(dele_tuple); + const auto file_count = std::get<2>(dele_tuple); + + auto hash_query_batch = parRedis.make_batch(); + for (FileIDType i = file_base_index; i < file_base_index + file_count; ++i) { + const auto file_key = PROGRAM_NAME ":file:" + lexical_cast(i); + hash_query_batch.hget(file_key, "hash"); + } + hash_query_batch.throw_if_failed(); + + for (const auto& rep : hash_query_batch.replies()) { + const auto hash_key = PROGRAM_NAME ":hash:" + redis::get_string(rep); + parDeleHash.run( + dele_batch.batch(), + std::make_tuple(hash_key), + std::make_tuple(lexical_cast(file_base_index), lexical_cast(file_count)) + ); + } + + dele_batch.del(PROGRAM_NAME ":set:" + lexical_cast(set_id)); + dele_batch.del(PROGRAM_NAME ":levels:" + lexical_cast(set_id)); + chunked_run(dele_batch.batch(), +"DEL", file_base_index, file_count, [](FileIDType id){return PROGRAM_NAME ":file:" + lexical_cast(id);}); + } + + dele_batch.throw_if_failed(); + } +} //namespace dindb diff --git a/src/backends/redis/delete.hpp b/src/backends/redis/delete.hpp new file mode 100644 index 0000000..aa07f1e --- /dev/null +++ b/src/backends/redis/delete.hpp @@ -0,0 +1,45 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id1E0F8FEB88EC4FED843FFEE7BAB624BB +#define id1E0F8FEB88EC4FED843FFEE7BAB624BB + +#include "backends/db_backend.hpp" +#include +#include +#include +#include + +namespace redis { + class IncRedis; + class Script; +} //namespace redis + +namespace dindb { + using IDDescMap = std::map; + using ConfirmDeleCallback = std::function; + + void delete_group_from_db ( + redis::IncRedis& parRedis, + redis::Script& parDeleTagIfInSet, + redis::Script& parDeleHash, + const std::vector& parIDs, + ConfirmDeleCallback parConf + ); +} //namespace dindb + +#endif diff --git a/src/backends/redis/find.cpp b/src/backends/redis/find.cpp new file mode 100644 index 0000000..4483517 --- /dev/null +++ b/src/backends/redis/find.cpp @@ -0,0 +1,331 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "find.hpp" +#include "incredis/incredis.hpp" +#include "duckhandy/lexical_cast.hpp" +#include "dindexerConfig.h" +#include "dindexer-core/split_tags.hpp" +#include "dindexer-machinery/tiger.hpp" +#include "duckhandy/compatibility.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dindb { + namespace { + inline std::string to_std_string ( boost::string_ref parStr ) a_always_inline; + template + inline void concatenate ( std::vector&& parAppend, std::vector& parOut ) a_always_inline; + template + inline T construct ( const std::vector& parData, const std::string& parID ) a_always_inline; + template <> + inline LocatedItem construct ( const std::vector& parData, const std::string& parID ) a_always_inline; + template <> + inline LocatedSet construct ( const std::vector& parData, const std::string& parID ) a_always_inline; + + bool all_tags_match (const TagList& parTags, const std::string& parTaglist) { + const auto tags = dincore::split_tags(parTaglist); + + if (tags.size() >= parTags.size()) { + for (const auto& required_tag : parTags) { + if (std::find(tags.begin(), tags.end(), required_tag) == tags.end()) { + return false; + } + } + } + else { + return false; + } + return true; + } + + //See: http://stackoverflow.com/questions/12552277/whats-the-best-way-to-iterate-over-two-or-more-containers-simultaneously/12553437#12553437 + //(referenced from http://stackoverflow.com/questions/16982190/c-use-boost-range-transformed-adaptor-with-binary-function) + //What became of this? http://marc.info/?l=boost-users&m=129619765731342 + template + auto zip_range (Conts&... parConts) -> decltype(boost::make_iterator_range( + boost::make_zip_iterator(boost::make_tuple(parConts.begin()...)), + boost::make_zip_iterator(boost::make_tuple(parConts.end()...))) + ) { + return { + boost::make_zip_iterator(boost::make_tuple(parConts.begin()...)), + boost::make_zip_iterator(boost::make_tuple(parConts.end()...)) + }; + } + + template + std::vector store_filtered_items ( + const std::vector& parReplies, + const std::vector& parIDs, + std::function, std::string>&)> parFilter + ) { + using boost::adaptors::filtered; + using boost::adaptors::transformed; + using boost::tuple; + using boost::make_tuple; + using redis::Reply; + using std::vector; + using dhandy::lexical_cast; + + assert(parReplies.size() == parIDs.size()); + return boost::copy_range>( + zip_range(parReplies, parIDs) | + transformed([](const tuple& r) { + return make_tuple(redis::get_array(r.get<0>()), r.get<1>()); + }) | + filtered(parFilter) | + transformed([](const tuple, std::string>& t) { + return construct(t.get<0>(), t.get<1>()); + }) + ); + } + + std::string to_std_string (boost::string_ref parStr) { + return std::string(parStr.data(), parStr.size()); + } + + template + void concatenate (std::vector&& parAppend, std::vector& parOut) { + parOut.insert(parOut.end(), std::make_move_iterator(parAppend.begin()), std::make_move_iterator(parAppend.end())); + } + + template <> + LocatedItem construct (const std::vector& parData, const std::string& parID) { + using dhandy::lexical_cast; + + //parData is expected to contain: "path", "group_id", "tags" + + assert(parData.size() == 3); + return LocatedItem{ + redis::get_string(parData[0]), + lexical_cast(parID), + lexical_cast(redis::get_string(parData[1])) + }; + } + + template <> + LocatedSet construct (const std::vector& parData, const std::string& parID) { + using dhandy::lexical_cast; + + //parData is expected to contain: "desc", "item_count", "dir_count" + + assert(parData.size() == 3); + const auto itm_count = lexical_cast(redis::get_string(parData[1])); + const auto dir_count = lexical_cast(redis::get_string(parData[2])); + assert(dir_count <= itm_count); + + return LocatedSet{ + redis::get_string(parData[0]), + lexical_cast(parID), + itm_count - dir_count, + dir_count + }; + } + + template + std::vector locate_in_bursts (redis::IncRedis& parRedis, const char* parScanKeyFilter, F parFilter, FIELDS&&... parFields) { + using dincore::split_and_trim; + + const int prefetch_count = 500; + std::vector retval; + std::vector ids; + ids.reserve(prefetch_count); + + int curr_count = 0; + auto batch = parRedis.make_batch(); + for (const auto& itm : parRedis.scan(parScanKeyFilter)) { + ++curr_count; + batch.hmget(itm, std::forward(parFields)...); + ids.push_back(to_std_string(split_and_trim(itm, ':').back())); + + if (curr_count == prefetch_count) { + concatenate(store_filtered_items(batch.replies(), ids, parFilter), retval); + batch.reset(); + curr_count = 0; + ids.clear(); + } + } + if (curr_count) + concatenate(store_filtered_items(batch.replies(), ids, parFilter), retval); + return retval; + } + } //unnamed namespace + + std::vector find_all_sets (redis::IncRedis& parRedis) { + using dincore::split_and_trim; + + std::vector retval; + for (const auto& itm : parRedis.scan(PROGRAM_NAME ":set:*")) { + retval.push_back(dhandy::lexical_cast(split_and_trim(itm, ':').back())); + } + return retval; + } + + std::vector locate_in_db (redis::IncRedis& parRedis, const std::string& parRegex, const TagList& parTags) { + const boost::regex search(parRegex, boost::regex_constants::optimize | boost::regex_constants::nosubs | boost::regex_constants::perl); + + auto filter = [&parTags, &search](const boost::tuple, std::string>& t) { + return (parTags.empty() or all_tags_match(parTags, redis::get_string(t.get<0>()[2]))) and + boost::regex_search(redis::get_string(t.get<0>()[0]), search); + }; + + return locate_in_bursts(parRedis, PROGRAM_NAME ":file:*", filter, "path", "group_id", "tags"); + } + + std::vector locate_in_db (redis::IncRedis& parRedis, const mchlib::TigerHash& parSearch, const TagList& parTags) { + const auto hash_key = PROGRAM_NAME ":hash:" + mchlib::tiger_to_string(parSearch, false); + const auto file_ids = parRedis.smembers(hash_key); + + std::vector ids; + if (file_ids) { + auto batch = parRedis.make_batch(); + for (auto&& file_id : *file_ids) { + if (not file_id) + continue; + + const auto file_key = PROGRAM_NAME ":file:" + *file_id; + ids.emplace_back(std::move(*file_id)); + batch.hmget(file_key, "path", "group_id", "tags"); + } + batch.throw_if_failed(); + + return store_filtered_items( + batch.replies(), + ids, + [&parTags](const boost::tuple, std::string>& t) { + return parTags.empty() or all_tags_match(parTags, redis::get_string(t.get<0>()[2])); + } + ); + } + else { + return std::vector(); + } + } + + std::vector locate_sets_in_db (redis::IncRedis& parRedis, const std::string& parSubstr, bool parCaseInsensitive) { + return locate_sets_in_db(parRedis, parSubstr, std::vector(), parCaseInsensitive); + } + + std::vector locate_sets_in_db (redis::IncRedis& parRedis, const std::string& parSubstr, const std::vector& parSets, bool parCaseInsensitive) { + using dhandy::lexical_cast; + + auto filter_case_ins = [&parSubstr, &parSets](const boost::tuple, std::string>& t) { + const auto& s = redis::get_string(t.get<0>()[0]); + return s.end() != std::search( + s.begin(), + s.end(), + parSubstr.begin(), + parSubstr.end(), + [](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);}) and + ( + parSets.empty() or + std::find(parSets.begin(), parSets.end(), lexical_cast(t.get<1>())) != parSets.end() + ); + }; + auto filter_case_sens = [&parSubstr, &parSets](const boost::tuple, std::string>& t) { + return redis::get_string(t.get<0>()[0]).find(parSubstr) != std::string::npos and + ( + parSets.empty() or + std::find(parSets.begin(), parSets.end(), lexical_cast(t.get<1>())) != parSets.end() + ); + }; + std::function, std::string>&)> filter; + if (parCaseInsensitive) + filter = filter_case_ins; + else + filter = filter_case_sens; + + return locate_in_bursts(parRedis, PROGRAM_NAME ":set:*", filter, "desc", "item_count", "dir_count"); + } + + std::vector> find_set_details (redis::IncRedis& parRedis, const std::vector& parSets) { + using dhandy::lexical_cast; + + auto batch = parRedis.make_batch(); + for (auto set_id : parSets) { + const auto set_key = PROGRAM_NAME ":set:" + lexical_cast(set_id); + batch.hmget(set_key, "creation", "name", "disk_label"); + } + batch.throw_if_failed(); + + std::vector> retval; + auto curr_set = parSets.begin(); + for (const auto& reply : batch.replies()) { + const auto& reply_list = get_array(reply); + if (redis::RedisVariantType_Nil != reply_list[0].which() and + redis::RedisVariantType_Nil != reply_list[1].which() and + redis::RedisVariantType_Nil != reply_list[2].which()) + { + retval.resize(retval.size() + 1); + retval.back().push_back(lexical_cast(*curr_set)); + retval.back().push_back(get_string(reply_list[1])); + retval.back().push_back(get_string(reply_list[0])); + retval.back().push_back(get_string(reply_list[2])); + } + ++curr_set; + } + return retval; + }; + + std::vector> find_file_details (redis::IncRedis& parRedis, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) { + using dhandy::lexical_cast; + using RetListType = std::vector>; + + const double level = static_cast(parLevel); + auto lst = parRedis.zrangebyscore(PROGRAM_NAME ":levels:" + lexical_cast(parSetID), level, true, level, true, false); + if (not lst) + return RetListType(); + + auto batch = parRedis.make_batch(); + for (const auto& itm : *lst) { + if (itm) + batch.hget(*itm, "path"); + } + batch.throw_if_failed(); + + std::vector> retval; + for (auto& reply : batch.replies()) { + if (redis::RedisVariantType_Nil != reply.which()) { + auto curr_path = get_string(reply); + if (boost::starts_with(curr_path, parDir)) { + retval.resize(retval.size() + 1); + retval.back().push_back(std::move(curr_path)); + } + } + } + return retval; + }; + + std::vector find_paths_starting_by (redis::IncRedis& parRedis, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) { + using boost::adaptors::transformed; + using dhandy::MaxSizedArray; + + auto file_details = find_file_details(parRedis, parGroupID, parLevel, parPath); + return boost::copy_range>( + file_details | + transformed([](MaxSizedArray& a){return std::move(a.front());}) + ); + } +} //namespace dindb diff --git a/src/backends/redis/find.hpp b/src/backends/redis/find.hpp new file mode 100644 index 0000000..209a768 --- /dev/null +++ b/src/backends/redis/find.hpp @@ -0,0 +1,43 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef idB4972996B4494E66A03B6AE205B1FA36 +#define idB4972996B4494E66A03B6AE205B1FA36 + +#include "backends/db_backend.hpp" +#include "duckhandy/MaxSizedArray.hpp" +#include +#include +#include +#include + +namespace redis { + class IncRedis; +} //namespace redis + +namespace dindb { + std::vector find_all_sets ( redis::IncRedis& parRedis ); + std::vector locate_in_db ( redis::IncRedis& parRedis, const std::string& parRegex, const TagList& parTags ); + std::vector locate_in_db ( redis::IncRedis& parRedis, const mchlib::TigerHash& parSearch, const TagList& parTags ); + std::vector locate_sets_in_db ( redis::IncRedis& parRedis, const std::string& parSubstr, bool parCaseInsensitive ); + std::vector locate_sets_in_db ( redis::IncRedis& parRedis, const std::string& parSubstr, const std::vector& parSets, bool parCaseInsensitive ); + std::vector> find_set_details ( redis::IncRedis& parRedis, const std::vector& parSets ); + std::vector> find_file_details ( redis::IncRedis& parRedis, GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir ); + std::vector find_paths_starting_by ( redis::IncRedis& parRedis, GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath ); +} //namespace dindb + +#endif diff --git a/src/backends/redis/record_data_adapt.hpp b/src/backends/redis/record_data_adapt.hpp new file mode 100644 index 0000000..f57b838 --- /dev/null +++ b/src/backends/redis/record_data_adapt.hpp @@ -0,0 +1,97 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id2949D72CC2F246D4A289FFB820CC3A8F +#define id2949D72CC2F246D4A289FFB820CC3A8F + +#include "dindexer-machinery/recorddata.hpp" +#include "duckhandy/lexical_cast.hpp" +#include +#include +#include +#include + +namespace dindb { + using FileRecordDataWithGroup = std::pair; +} //namespace dindb + +namespace redis { + template <> + struct StructAdapt { + template + static bool decode (const R& parRange, dindb::FileRecordDataWithGroup& parOut) { + using dhandy::lexical_cast; + + std::array mime; + std::string group_key; + + for (const auto itm : parRange) { + if (itm.first == "path") + parOut.second.abs_path = itm.second; + else if (itm.first == "hash") + parOut.second.hash = mchlib::string_to_tiger(itm.second); + else if (itm.first == "size") + parOut.second.size = lexical_cast(itm.second); + else if (itm.first == "level") + parOut.second.level = lexical_cast(itm.second); + else if (itm.first == "mime_type") + mime[0] = itm.second; + else if (itm.first == "mime_charset") + mime[1] = itm.second; + else if (itm.first == "is_directory") + parOut.second.is_directory = (itm.second[0] == '0' ? false : true); + else if (itm.first == "is_symlink") + parOut.second.is_symlink = (itm.second[0] == '0' ? false : true); + else if (itm.first == "unreadable") + parOut.second.unreadable = (itm.second[0] == '0' ? false : true); + else if (itm.first == "hash_valid") + parOut.second.hash_valid = (itm.second[0] == '0' ? false : true); + else if (itm.first == "group_id") + parOut.first = itm.second; + } + parOut.second.mime_full = mime[0] + mime[1]; + parOut.second.mime_type_offset = 0; + parOut.second.mime_type_length = parOut.second.mime_charset_offset = static_cast(mime[0].size()); + parOut.second.mime_charset_length = static_cast(mime[1].size()); + return true; + } + }; + + template <> + struct StructAdapt { + template + static bool decode (const R& parRange, mchlib::SetRecordDataFull& parOut) { + using dhandy::lexical_cast; + + for (const auto& itm : parRange) { + if (itm.first == "name") + parOut.name = itm.second; + else if (itm.first == "disk_label") + parOut.disk_label = itm.second; + else if (itm.first == "fs_uuid") + parOut.fs_uuid = itm.second; + else if (itm.first == "type") + parOut.type = itm.second[0]; + else if (itm.first == "content_type") + parOut.content_type = itm.second[0]; + } + return true; + } + }; +} //namespace redis + +#endif diff --git a/include/helpers/stringize.h b/src/backends/redis/redisConfig.h.in similarity index 82% rename from include/helpers/stringize.h rename to src/backends/redis/redisConfig.h.in index b007909..6f1267d 100644 --- a/include/helpers/stringize.h +++ b/src/backends/redis/redisConfig.h.in @@ -15,10 +15,9 @@ * along with "dindexer". If not, see . */ -#ifndef id8B9FF1807C05459DAE8CD05312147416 -#define id8B9FF1807C05459DAE8CD05312147416 +#ifndef idB3E2F339E3B64378B66342550C4D2089 +#define idB3E2F339E3B64378B66342550C4D2089 -#define STRINGIZE_IMPL(s) #s -#define STRINGIZE(s) STRINGIZE_IMPL(s) +#define REDIS_SCRIPTS_PATH "@DINDEXER_REDIS_SCRIPTS_PATH@" #endif diff --git a/src/backends/redis/tag.cpp b/src/backends/redis/tag.cpp new file mode 100644 index 0000000..d416db2 --- /dev/null +++ b/src/backends/redis/tag.cpp @@ -0,0 +1,193 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "tag.hpp" +#include "incredis/incredis.hpp" +#include "dindexerConfig.h" +#include "duckhandy/lexical_cast.hpp" +#include "dindexer-core/split_tags.hpp" +#include +#include +#include +#include +#include + +namespace std { + template<> + struct hash { + size_t operator() (boost::string_ref const& sr) const { + return boost::hash_range(sr.begin(), sr.end()); + } + }; +} //namespace std + +namespace dindb { + namespace { + std::string make_file_key (FileIDType parID) { + return PROGRAM_NAME ":file:" + dhandy::lexical_cast(parID); + } + + std::vector compile_regexes (const std::vector& parRegexes) { + std::vector retval; + retval.reserve(parRegexes.size()); + for (const auto& str : parRegexes) { + retval.emplace_back(boost::regex( + str, + boost::regex_constants::optimize | boost::regex_constants::nosubs | boost::regex_constants::perl + )); + } + assert(retval.size() == parRegexes.size()); + return retval; + } + + void run_id_based_script (redis::IncRedis& parRedis, redis::Script& parScript, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + using dhandy::lexical_cast; + + auto batch = parRedis.command().make_batch(); + const std::string set_id = lexical_cast(parSet); + for (const auto file_id : parFiles) { + for (const auto &tag : parTags) { + std::ostringstream oss; + oss << PROGRAM_NAME ":tag:" << tag; + const std::string tag_key = oss.str(); + const std::string file_key = make_file_key(file_id); + parScript.run(batch, std::make_tuple(tag_key, file_key), std::make_tuple(set_id)); + } + } + + batch.throw_if_failed(); + } + + void run_regex_based_script(redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + using dhandy::lexical_cast; + + const std::string set_id = lexical_cast(parSet); + const auto regexes = compile_regexes(parRegexes); + for (const auto &itm : parRedis.scan(PROGRAM_NAME ":file:*")) { + const auto &file_key = itm; + const auto path = parRedis.hget(file_key, "path"); + + auto batch = parRedis.command().make_batch(); + for (const auto ®ex : regexes) { + if (not path or not boost::regex_search(*path, regex)) + continue; + + for (const auto &tag : parTags) { + std::ostringstream oss; + oss << PROGRAM_NAME ":tag:" << tag; + const std::string tag_key = oss.str(); + parTagIfInSet.run(batch, std::make_tuple(tag_key, file_key), std::make_tuple(set_id)); + } + break; + } + batch.throw_if_failed(); + } + } + + template + T id_from_redis_key (const std::string& parKey) { + assert(not parKey.empty()); + return dhandy::lexical_cast(dincore::split_and_trim(parKey, ':').back()); + } + } //unnamed namespace + + void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + run_id_based_script(parRedis, parTagIfInSet, parFiles, parTags, parSet); + } + + void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + run_regex_based_script(parRedis, parTagIfInSet, parRegexes, parTags, parSet); + } + + void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { + run_id_based_script(parRedis, parDeleIfInSet, parFiles, parTags, parSet); + } + + void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + run_regex_based_script(parRedis, parDeleIfInSet, parRegexes, parTags, parSet); + } + + void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector& parFiles, GroupIDType parSet) { + auto batch = parRedis.make_batch(); + for (const auto file_id : parFiles) { + const auto file_key = make_file_key(file_id); + batch.hget(file_key, "tags"); + } + + batch.throw_if_failed(); + std::unordered_set dele_tags; + for (const auto& reply : batch.replies()) { + auto tags = dincore::split_tags(redis::get_string(reply)); + for (const auto& tag : tags) { + dele_tags.insert(tag); + } + } + + std::vector vec_dele_tags(dele_tags.begin(), dele_tags.end()); + delete_tags(parRedis, parDeleIfInSet, parFiles, vec_dele_tags, parSet); + } + + void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector& parRegexes, GroupIDType parSet) { + using dhandy::lexical_cast; + + const auto regexes = compile_regexes(parRegexes); + + std::unordered_set dele_tags; + std::vector ids; + + for (const auto& itm : parRedis.scan(PROGRAM_NAME ":file:*")) { + const auto& file_key = itm; + auto opt_file_replies = parRedis.hmget(file_key, "path", "tags", "group_id"); + assert(opt_file_replies and opt_file_replies->size() == 3); + if (not opt_file_replies) + continue; + const auto& file_replies = *opt_file_replies; + const auto group_id = lexical_cast(*file_replies[2]); + if (parSet != InvalidGroupID and parSet != group_id) + continue; + + const auto& path = *file_replies[0]; + const auto tags_str = (file_replies[1] ? std::string() : *file_replies[1]); + const auto tags = dincore::split_tags(tags_str); + const auto file_id = id_from_redis_key(file_key); + + for (const auto ®ex : regexes) { + if (not boost::regex_search(path, regex)) + continue; + + ids.push_back(file_id); + + for (const auto &tag : tags) { + dele_tags.insert(std::string(tag.data(), tag.size())); + } + break; + } + } + + std::vector dele_tags_vec; + dele_tags_vec.reserve(dele_tags.size()); + std::transform( + dele_tags.begin(), + dele_tags.end(), + std::back_inserter(dele_tags_vec), + [](const std::string& parS) { return boost::string_ref(parS); } + ); + + delete_tags(parRedis, parDeleIfInSet, ids, dele_tags_vec, parSet); + } +} //namespace dindb + diff --git a/src/backends/redis/tag.hpp b/src/backends/redis/tag.hpp new file mode 100644 index 0000000..3f4acd3 --- /dev/null +++ b/src/backends/redis/tag.hpp @@ -0,0 +1,74 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id2874EA620CF0415CAF5B005E227BC44B +#define id2874EA620CF0415CAF5B005E227BC44B + +#include "backends/db_backend.hpp" +#include +#include + +namespace redis { + class IncRedis; + class Script; +} //namespace redis + +namespace dindb { + void tag_files ( + redis::IncRedis& parRedis, + redis::Script& parTagIfInSet, + const std::vector& parFiles, + const std::vector& parTags, + GroupIDType parSet + ); + void tag_files ( + redis::IncRedis& parRedis, + redis::Script& parTagIfInSet, + const std::vector& parRegexes, + const std::vector& parTags, + GroupIDType parSet + ); + + void delete_tags ( + redis::IncRedis& parRedis, + redis::Script& parDeleIfInSet, + const std::vector& parFiles, + const std::vector& parTags, + GroupIDType parSet + ); + void delete_tags ( + redis::IncRedis& parRedis, + redis::Script& parDeleIfInSet, + const std::vector& parRegexes, + const std::vector& parTags, + GroupIDType parSet + ); + void delete_all_tags ( + redis::IncRedis& parRedis, + redis::Script& parDeleIfInSet, + const std::vector& parFiles, + GroupIDType parSet + ); + void delete_all_tags ( + redis::IncRedis& parRedis, + redis::Script& parDeleIfInSet, + const std::vector& parRegexes, + GroupIDType parSet + ); +} //namespace dindb + +#endif diff --git a/src/backends/redis/tag_if_in_set.lua b/src/backends/redis/tag_if_in_set.lua new file mode 100644 index 0000000..2fc166f --- /dev/null +++ b/src/backends/redis/tag_if_in_set.lua @@ -0,0 +1,65 @@ +-- Copyright 2015, 2016, Michele Santullo +-- This file is part of "dindexer". +-- +-- "dindexer" is free software: you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- "dindexer" is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with "dindexer". If not, see . + +local tag_key = KEYS[1] +local file_key = KEYS[2] +local group_id = ARGV[1] + +local function split_string(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t = {} + local z = 1 + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + t[z] = str + z = z + 1 + end + return t +end + +local function add_tag_to_list(tag_list, new_tag) + if tag_list == false or tag_list == "" then + return new_tag + end + tag_list = split_string(tag_list, ",") + + local new_tag_found = false + for _,str in ipairs(tag_list) do + if str == new_tag then + new_tag_found = true + break + end + end + if not new_tag_found then + tag_list[#tag_list+1] = new_tag + end + table.sort(tag_list) + return table.concat(tag_list, ",") +end + +if group_id ~= 0 then + local found_group_id = redis.call("HGET", file_key, "group_id") + if found_group_id ~= group_id then + return nil + end +end + +local new_tag = split_string(tag_key, ":") +local new_tag_list = add_tag_to_list(redis.call("HGET", file_key, "tags"), new_tag[#new_tag]) + +redis.call("HSET", file_key, "tags", new_tag_list) +return redis.call("SADD", tag_key, file_key) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 316497a..aaa2809 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -project(${bare_name}-common CXX C) +project(${bare_name}-common CXX) find_package(Readline 6.3 REQUIRED) @@ -8,7 +8,6 @@ add_library(${PROJECT_NAME} validationerror.cpp common_info.cpp readline_wrapper.cpp - split_tags.cpp ) target_include_directories(${PROJECT_NAME} @@ -23,6 +22,8 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ${bare_name}-if PRIVATE ${YAMLCPP_LIBRARY} PRIVATE ${Readline_LIBRARY} + ${bare_name}-backend + ${bare_name}-core ) #install(TARGETS ${PROJECT_NAME} @@ -31,7 +32,7 @@ target_link_libraries(${PROJECT_NAME} # ARCHIVE DESTINATION lib/static #) -#Allow to link with .so +#Allow linking with .so #see https://cmake.org/pipermail/cmake/2007-May/014350.html #and http://stackoverflow.com/questions/6093547/what-do-r-x86-64-32s-and-r-x86-64-64-relocation-mean/6093910#6093910 if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" ) diff --git a/src/common/commandline.cpp b/src/common/commandline.cpp index 74e9277..2fd86b6 100644 --- a/src/common/commandline.cpp +++ b/src/common/commandline.cpp @@ -18,9 +18,9 @@ #include "dindexer-common/commandline.hpp" #include "dindexerConfig.h" #include "gitinfo.h" -#include "helpers/lengthof.h" -#include "helpers/stringize.h" -#include "helpers/infix_iterator.hpp" +#include "duckhandy/lengthof.h" +#include "duckhandy/stringize.h" +#include "duckhandy/infix_iterator.hpp" #include #include #include diff --git a/src/common/common_info.cpp b/src/common/common_info.cpp index 8ed97d5..e3d1173 100644 --- a/src/common/common_info.cpp +++ b/src/common/common_info.cpp @@ -17,7 +17,7 @@ #include "dindexer-common/common_info.hpp" #include "dindexerConfig.h" -#include "helpers/stringize.h" +#include "duckhandy/stringize.h" namespace dinlib { namespace { diff --git a/src/common/settings.cpp b/src/common/settings.cpp index e37c189..bdb66d2 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -16,59 +16,46 @@ */ #include "dindexer-common/settings.hpp" +#include "dindexer-core/split_tags.hpp" +#include "dindexer-core/searchpaths.hpp" +#include "dindexerConfig.h" #include #include #include - -namespace YAML { - template<> - struct convert { - static Node encode (const dinlib::SettingsDB& parSettings) { - Node node; - node["address"] = parSettings.address; - node["username"] = parSettings.username; - node["password"] = parSettings.password; - node["port"] = parSettings.port; - node["dbname"] = parSettings.dbname; - return node; - } - - static bool decode (const Node& parNode, dinlib::SettingsDB& parSettings) { - if (not parNode.IsMap() or parNode.size() != 5) { - return false; - } - - parSettings.address = parNode["address"].as(); - parSettings.username = parNode["username"].as(); - parSettings.password = parNode["password"].as(); - parSettings.dbname = parNode["dbname"].as(); - parSettings.port = parNode["port"].as(); - return true; - } - }; -} //namespace YAML +#include +#include +#include +#include namespace dinlib { namespace { std::string expand ( const char* parString ); + std::string find_plugin_by_name ( std::vector&& parSearchPaths, const std::string& parName ); + void throw_if_plugin_failed ( const dindb::BackendPlugin& parPlugin, const std::string& parPluginPath, const std::string& parIntendedName ); } //unnamed namespace - bool load_settings (const std::string& parPath, dinlib::Settings& parOut, bool parExpand) { + void load_settings (const std::string& parPath, dinlib::Settings& parOut, bool parExpand) { + using dincore::split_and_trim; + const std::string path = (parExpand ? expand(parPath.c_str()) : parPath); + std::string search_paths; - try { - auto settings = YAML::LoadFile(path); + auto settings = YAML::LoadFile(path); - if (settings["db_settings"]) { - parOut.db = settings["db_settings"].as(); - return true; - } + if (not settings["backend_name"]) + throw std::runtime_error("No backend_name given in the config file"); + if (settings["backend_paths"]) + search_paths += ":" + settings["backend_paths"].as(); + parOut.backend_name = settings["backend_name"].as(); + const std::string backend_settings_section = parOut.backend_name + "_settings"; + if (settings[backend_settings_section]) { + auto settings_node = settings[backend_settings_section]; + const std::string plugin_path = find_plugin_by_name(split_and_trim(search_paths, ':'), parOut.backend_name); + if (plugin_path.empty()) + throw std::runtime_error(std::string("Unable to find any suitable plugin with the specified name \"") + parOut.backend_name + "\""); + parOut.backend_plugin = dindb::BackendPlugin(plugin_path, &settings_node); + throw_if_plugin_failed(parOut.backend_plugin, plugin_path, parOut.backend_name); } - catch (const std::exception&) { - return false; - } - - return false; } namespace { @@ -83,5 +70,39 @@ namespace dinlib { wordfree(&p); return oss.str(); } + + std::string find_plugin_by_name (std::vector&& parSearchPaths, const std::string& parName) { + dincore::ShallowSearchPaths search_paths(std::move(parSearchPaths)); + return search_paths.first_hit( + [&parName](boost::string_ref, const std::string& parPath) { + return dindb::backend_name(parPath) == parName; + }, + dincore::SearchPaths::File + ); + } + + void throw_if_plugin_failed (const dindb::BackendPlugin& parPlugin, const std::string& parPluginPath, const std::string& parIntendedName) { + if (not parPlugin.is_loaded()) { + std::ostringstream oss; + oss << "Unable to load plugin \"" << parIntendedName << + "\" found at path \"" << parPluginPath << '"'; + throw std::runtime_error(oss.str()); + } + if (parPlugin.name() != parIntendedName) { + std::ostringstream oss; + oss << "Plugin \"" << parIntendedName << "\" not found." << + " Plugin at path \"" << parPluginPath << "\" reports \"" << + parPlugin.name() << "\" as its name"; + throw std::runtime_error(oss.str()); + } + if (parPlugin.max_supported_interface_version() < parPlugin.backend_interface_version()) { + std::ostringstream oss; + oss << "Plugin \"" << parPlugin.name() << "\" at path \"" << + parPluginPath << "\" uses interface version " << parPlugin.backend_interface_version() << + " but the maximum supported interface version is " << + parPlugin.max_supported_interface_version(); + throw std::runtime_error(oss.str()); + } + } } //unnamed namespace } //namespace dinlib diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt new file mode 100644 index 0000000..db7fa9b --- /dev/null +++ b/src/core/CMakeLists.txt @@ -0,0 +1,36 @@ +project(${bare_name}-core CXX) + +add_library(${PROJECT_NAME} SHARED + searchpaths.cpp + split_tags.cpp +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE ${bare_name}-if +) + +target_include_directories(${PROJECT_NAME} + PRIVATE ${DINDEXER_PUB_INCLUDE_DIR}/${bare_name}-core +) + +target_compile_features(${PROJECT_NAME} + INTERFACE cxx_nullptr + INTERFACE cxx_range_for + INTERFACE cxx_lambdas + INTERFACE cxx_decltype_auto + INTERFACE cxx_defaulted_functions + INTERFACE cxx_deleted_functions + INTERFACE cxx_auto_type + INTERFACE cxx_decltype_incomplete_return_types + INTERFACE cxx_defaulted_move_initializers + INTERFACE cxx_noexcept + INTERFACE cxx_rvalue_references + INTERFACE cxx_generalized_initializers + INTERFACE cxx_variadic_templates +) + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + ARCHIVE DESTINATION lib/static +) diff --git a/src/core/searchpaths.cpp b/src/core/searchpaths.cpp new file mode 100644 index 0000000..573361b --- /dev/null +++ b/src/core/searchpaths.cpp @@ -0,0 +1,136 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "searchpaths.hpp" +#include +#include +#include +#include +#include + +namespace dincore { + namespace { + const std::string& STR_to_string (const std::string& parString) { + return parString; + } + + std::string STR_to_string (boost::string_ref parString) { + return std::string(parString.data(), parString.size()); + } + + template + std::string first_hit_impl (const std::vector& parPaths, SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType) { + using boost::filesystem::path; + using boost::filesystem::is_directory; + using boost::filesystem::directory_iterator; + using boost::filesystem::directory_entry; + using boost::make_iterator_range; + + for (const auto& curr_dir_path : parPaths) { + path curr_path(STR_to_string(curr_dir_path)); + auto listing = make_iterator_range(directory_iterator(curr_path), directory_iterator()); + for (const directory_entry& entry : listing) { + if ( + (parType == SearchPaths::Any) or + (parType == SearchPaths::Directory and is_directory(entry)) or + (parType == SearchPaths::File and not is_directory(entry)) + ) { + auto str_path = entry.path().string(); + if (parPredicate(curr_dir_path, str_path)) + return str_path; + } + } + } + return std::string(); + } + + //std::string make_file_path (boost::string_ref parPath, boost::string_ref parName) { + // assert(not parName.empty()); + // if (parName.empty()) + // return std::string(parPath.data(), parPath.size()); + + // std::string retval; + // const std::size_t slash = (not parPath.empty() and parPath[parPath.size() - 1] != '/' ? 1 : 0); + // retval.reserve(parPath.size() + parName.size() + slash); + // std::copy(parPath.begin(), parPath.end(), retval.begin()); + // retval[parPath.size()] = '/'; + // std::copy(parName.begin(), parName.end(), retval.begin() + parPath.size() + slash); + // return retval; + //} + + bool is_same_filename (boost::string_ref parBaseDir, const std::string& parFullPath, boost::string_ref parFilename) { + assert(parBaseDir.size() < parFullPath.size()); + return boost::string_ref(parFullPath).substr(parBaseDir.size() + 1) == parFilename; + } + } //unnamed namespace + + SearchPaths::SearchPaths (std::vector&& parList) : + m_paths(std::move(parList)) + { + } + + SearchPaths::SearchPaths (std::initializer_list parInit) : + m_paths(parInit) + { + } + + SearchPaths::~SearchPaths() noexcept = default; + + void SearchPaths::add_path (std::string&& parPath) { + if (std::find(m_paths.begin(), m_paths.end(), parPath) == m_paths.end()) { + m_paths.emplace_back(std::move(parPath)); + } + } + + std::string SearchPaths::first_hit (boost::string_ref parFile, SearchType parType) const { + using std::placeholders::_1; + using std::placeholders::_2; + return first_hit_impl(m_paths, std::bind(&is_same_filename, _1, _2, parFile), parType); + } + + std::string SearchPaths::first_hit (Predicate parPredicate, SearchType parType) const { + return first_hit_impl(m_paths, parPredicate, parType); + } + + ShallowSearchPaths::ShallowSearchPaths (std::vector&& parList) : + m_paths(std::move(parList)) + { + } + + ShallowSearchPaths::ShallowSearchPaths (std::initializer_list parInit) : + m_paths(parInit) + { + } + + ShallowSearchPaths::~ShallowSearchPaths() noexcept = default; + + void ShallowSearchPaths::add_path (boost::string_ref parPath) { + if (std::find(m_paths.begin(), m_paths.end(), parPath) == m_paths.end()) { + m_paths.push_back(parPath); + } + } + + std::string ShallowSearchPaths::first_hit (boost::string_ref parFile, SearchPaths::SearchType parType) const { + using std::placeholders::_1; + using std::placeholders::_2; + return first_hit_impl(m_paths, std::bind(&is_same_filename, _1, _2, parFile), parType); + } + + std::string ShallowSearchPaths::first_hit (SearchPaths::Predicate parPredicate, SearchPaths::SearchType parType) const { + return first_hit_impl(m_paths, parPredicate, parType); + } +} //namespace dincore diff --git a/src/common/split_tags.cpp b/src/core/split_tags.cpp similarity index 85% rename from src/common/split_tags.cpp rename to src/core/split_tags.cpp index 4ba7275..cdc5ef9 100644 --- a/src/common/split_tags.cpp +++ b/src/core/split_tags.cpp @@ -15,7 +15,7 @@ * along with "dindexer". If not, see . */ -#include "dindexer-common/split_tags.hpp" +#include "dindexer-core/split_tags.hpp" #include #include #include @@ -24,8 +24,12 @@ #include #include -namespace dinlib { +namespace dincore { std::vector split_tags (const std::string& parCommaSeparatedList) { + return split_and_trim(parCommaSeparatedList, ','); + } + + std::vector split_and_trim (const std::string& parList, char parSeparator) { using OutRange = boost::iterator_range; using boost::token_finder; using boost::adaptors::transformed; @@ -42,10 +46,10 @@ namespace dinlib { //http://www.boost.org/doc/libs/1_60_0/doc/html/boost/algorithm/token_finder.html //https://stackoverflow.com/questions/20781090/difference-between-boostsplit-vs-boostiter-split return boost::copy_range>( - iter_split(out_range, parCommaSeparatedList, token_finder([](char c){return ','==c;})) | + iter_split(out_range, parList, token_finder([parSeparator](char c){return parSeparator==c;})) | transformed([](const OutRange& r){return trim_copy(r);}) | transformed([](const OutRange& r){return string_ref(&*r.begin(), r.size());}) | filtered([](const string_ref& r){return not r.empty();}) ); } -} //namespace dinlib +} //namespace dincore diff --git a/src/delete/CMakeLists.txt b/src/delete/CMakeLists.txt index 7d05e34..88b71ad 100644 --- a/src/delete/CMakeLists.txt +++ b/src/delete/CMakeLists.txt @@ -3,7 +3,6 @@ project(${bare_name}-delete CXX) add_executable(${PROJECT_NAME} main.cpp commandline.cpp - postgre_delete.cpp ) target_include_directories(${PROJECT_NAME} diff --git a/src/delete/main.cpp b/src/delete/main.cpp index c510d6f..1e458ab 100644 --- a/src/delete/main.cpp +++ b/src/delete/main.cpp @@ -18,7 +18,6 @@ #include "commandline.hpp" #include "dindexer-common/settings.hpp" #include "dindexerConfig.h" -#include "postgre_delete.hpp" #include #include #include @@ -26,7 +25,7 @@ #include namespace { - bool confirm_delete (const din::IDDescMap& parMap) { + bool confirm_delete (const dindb::IDDescMap& parMap) { for (const auto& itm : parMap) { std::cout << "ID " << itm.first << '\t' << itm.second << '\n'; } @@ -38,7 +37,7 @@ namespace { return (answer.empty() or "y" == answer or "Y" == answer); } - bool always_delete (const din::IDDescMap&) { + bool always_delete (const dindb::IDDescMap&) { return true; } } //unnamed namespace @@ -62,12 +61,13 @@ int main (int parArgc, char* parArgv[]) { } dinlib::Settings settings; - { - const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings); - if (not loaded) { - std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n"; - return 1; - } + try { + dinlib::load_settings(CONFIG_FILE_PATH, settings); + } + catch (const std::runtime_error& err) { + std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n"; + std::cerr << err.what() << '\n'; + return 1; } if (not vm.count("groupid")) { @@ -77,7 +77,7 @@ int main (int parArgc, char* parArgv[]) { const auto ids = vm["groupid"].as>(); auto confirm_func = (vm.count("confirm") ? &always_delete : &confirm_delete); - din::delete_group_from_db(settings.db, ids, confirm_func); + settings.backend_plugin.backend().delete_group(ids, confirm_func); return 0; } diff --git a/src/dindexerConfig.h.in b/src/dindexerConfig.h.in index 832d551..61716d8 100644 --- a/src/dindexerConfig.h.in +++ b/src/dindexerConfig.h.in @@ -18,7 +18,7 @@ #ifndef id48D6E1D45238460F99C2BCBFDE920791 #define id48D6E1D45238460F99C2BCBFDE920791 -#include "dindexer-common/cmake_on_off.h" +#include "duckhandy/cmake_on_off.h" #define PROGRAM_NAME "@bare_name@" #define VERSION_MAJOR @PROJECT_VERSION_MAJOR@ diff --git a/src/locate/CMakeLists.txt b/src/locate/CMakeLists.txt index 9b68d0f..3ace773 100644 --- a/src/locate/CMakeLists.txt +++ b/src/locate/CMakeLists.txt @@ -3,7 +3,6 @@ project(${bare_name}-locate CXX) add_executable(${PROJECT_NAME} main.cpp commandline.cpp - postgre_locate.cpp hash.cpp ) diff --git a/src/locate/main.cpp b/src/locate/main.cpp index 7b619c4..6e8fa91 100644 --- a/src/locate/main.cpp +++ b/src/locate/main.cpp @@ -16,9 +16,8 @@ */ #include "commandline.hpp" -#include "postgre_locate.hpp" #include "dindexer-common/settings.hpp" -#include "dindexer-common/split_tags.hpp" +#include "dindexer-core/split_tags.hpp" #include "dindexerConfig.h" #include "hash.hpp" #include "glob2regex/glob2regex.hpp" @@ -27,7 +26,7 @@ #include #include -namespace din { +namespace dindb { std::ostream& operator<< (std::ostream& parStream, const LocatedItem& parItem) { parStream << parItem.group_id << '\t' << parItem.id << '\t' << parItem.path; return parStream; @@ -41,14 +40,14 @@ namespace din { '\t' << parItem.files_count << '\t' << dircount; return parStream; } -} //namespace din +} //namespace dindb namespace { std::vector extract_tags (const boost::program_options::variables_map& parVM) { if (not parVM.count("tags")) return std::vector(); else - return dinlib::split_tags(parVM["tags"].as()); + return dincore::split_tags(parVM["tags"].as()); } } //unnamed namespace @@ -72,31 +71,33 @@ int main (int parArgc, char* parArgv[]) { } dinlib::Settings settings; - { - const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings); - if (not loaded) { - std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n"; - return 1; - } + try { + dinlib::load_settings(CONFIG_FILE_PATH, settings); + } + catch (const std::runtime_error& err) { + std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n"; + std::cerr << err.what() << '\n'; + return 1; } + auto& db = settings.backend_plugin.backend(); if (vm.count("set")) { - const auto results = din::locate_sets_in_db(settings.db, vm["substring"].as(), not not vm.count("case-insensitive")); - std::copy(results.begin(), results.end(), std::ostream_iterator(std::cout, "\n")); + const auto results = db.locate_sets_in_db(vm["substring"].as(), not not vm.count("case-insensitive")); + std::copy(results.begin(), results.end(), std::ostream_iterator(std::cout, "\n")); } else { - std::vector results; + std::vector results; const std::vector tags = extract_tags(vm); if (vm.count("byhash")) { const auto hash = din::hash(vm["substring"].as()); - results = din::locate_in_db(settings.db, hash, tags); + results = db.locate_in_db(hash, tags); } else { const auto search_regex = g2r::convert(vm["substring"].as(), not vm.count("case-insensitive")); - results = din::locate_in_db(settings.db, search_regex, tags); + results = db.locate_in_db(search_regex, tags); } - std::copy(results.begin(), results.end(), std::ostream_iterator(std::cout, "\n")); + std::copy(results.begin(), results.end(), std::ostream_iterator(std::cout, "\n")); } return 0; } diff --git a/src/locate/postgre_locate.hpp b/src/locate/postgre_locate.hpp deleted file mode 100644 index b30c6fb..0000000 --- a/src/locate/postgre_locate.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef id1AE05A59AE0E4A4490040FD85D9AF665 -#define id1AE05A59AE0E4A4490040FD85D9AF665 - -#include "dindexer-common/settings.hpp" -#include -#include -#include -#include - -namespace mchlib { - struct TigerHash; -} //namespace mchlib - -namespace din { - struct LocatedItem { - std::string path; - uint64_t id; - uint32_t group_id; - }; - - struct LocatedSet { - std::string desc; - uint32_t id; - uint32_t files_count; - uint32_t dir_count; - }; - - using TagList = std::vector; - - std::vector locate_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, const TagList& parTags ); - std::vector locate_in_db ( const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags ); - std::vector locate_sets_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, bool parCaseInsensitive ); - std::vector locate_sets_in_db ( const dinlib::SettingsDB& parDB, const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive ); -} //namespace din - -#endif diff --git a/src/machinery/CMakeLists.txt b/src/machinery/CMakeLists.txt index 64c87c6..3b32b59 100644 --- a/src/machinery/CMakeLists.txt +++ b/src/machinery/CMakeLists.txt @@ -26,9 +26,8 @@ add_library(${PROJECT_NAME} SHARED scantask/setbasic.cpp ) -target_include_directories(${PROJECT_NAME} +target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${MAGIC_INCLUDE_DIR} - PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ) target_link_libraries(${PROJECT_NAME} @@ -37,6 +36,7 @@ target_link_libraries(${PROJECT_NAME} ) target_include_directories(${PROJECT_NAME} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PUBLIC ${CMAKE_SOURCE_DIR}/lib/better-enums ) diff --git a/src/machinery/discinfo.cpp b/src/machinery/discinfo.cpp index 3993c30..65bd2c7 100644 --- a/src/machinery/discinfo.cpp +++ b/src/machinery/discinfo.cpp @@ -17,7 +17,7 @@ #include "discinfo.hpp" #include "pathname.hpp" -#include "helpers/lengthof.h" +#include "duckhandy/lengthof.h" #include #include #include @@ -35,6 +35,10 @@ # include #endif #include +#include +#include + +namespace fs = boost::filesystem; namespace mchlib { namespace { @@ -167,6 +171,41 @@ namespace mchlib { return static_cast(retval); } #endif + + std::string find_with_same_inode (const std::string& parWhat, const char* parWhere) { + using boost::make_iterator_range; + + struct stat st; + if (stat(parWhat.c_str(), &st)) + return std::string(); + + const auto& inode = st.st_ino; + + fs::path p(parWhere); + if (not fs::exists(p) or not fs::is_directory(p)) { + throw std::runtime_error( + std::string("Search path \"") + p.string() + + "\" doesn't exist"); + } + + for (const fs::directory_entry& itm : make_iterator_range(fs::directory_iterator(p), fs::directory_iterator())) { + struct stat curr_st; + if (not stat(itm.path().c_str(), &curr_st) and inode == curr_st.st_ino) + return fs::basename(itm); + } + + return std::string(); + } + + //Get disc label by doing the equivalent of: + //find -L /dev/disk/by-label -inum $(stat -c %i /dev/sda1) -print + std::string retrieve_label (const std::string& parDev) { + return find_with_same_inode(parDev, "/dev/disk/by-label"); + } + + std::string retrieve_uuid (const std::string& parDev) { + return find_with_same_inode(parDev, "/dev/disk/by-uuid"); + } } //unnamed namespace DiscInfo::DiscInfo (std::string&& parPath) : @@ -187,6 +226,11 @@ namespace mchlib { } input_path.pop_right(); } while(input_path.atom_count() > 0); + + if (mountpoint_found()) { + m_label = retrieve_label(m_device); + m_uuid = retrieve_uuid(m_device); + } } bool DiscInfo::mountpoint_found() const { @@ -329,4 +373,12 @@ namespace mchlib { }; } #endif + + const std::string& DiscInfo::label() const { + return m_label; + } + + const std::string& DiscInfo::filesystem_uuid() const { + return m_uuid; + } } //namespace mchlib diff --git a/src/machinery/discinfo.hpp b/src/machinery/discinfo.hpp index b4b4e25..0093d1f 100644 --- a/src/machinery/discinfo.hpp +++ b/src/machinery/discinfo.hpp @@ -52,11 +52,15 @@ namespace mchlib { OpticalTypes optical_type ( void ) const; DriveTypes drive_type ( void ) const; #endif + const std::string& label ( void ) const; + const std::string& filesystem_uuid ( void ) const; private: const std::string m_initial_path; std::string m_mountpoint; std::string m_device; + std::string m_label; + std::string m_uuid; }; } //namespace mchlib diff --git a/src/machinery/machineryConfig.h.in b/src/machinery/machineryConfig.h.in index e33c358..84a46de 100644 --- a/src/machinery/machineryConfig.h.in +++ b/src/machinery/machineryConfig.h.in @@ -18,7 +18,7 @@ #ifndef id32A00B72E29D463795CBF66517387E8D #define id32A00B72E29D463795CBF66517387E8D -#include "dindexer-common/cmake_on_off.h" +#include "duckhandy/cmake_on_off.h" #if CMAKE_@DINDEXER_WITH_NICE_MEDIA_TYPES@ # define WITH_NICE_MEDIA_TYPES diff --git a/src/machinery/machinery_info.cpp b/src/machinery/machinery_info.cpp index d5c2089..6d7e17a 100644 --- a/src/machinery/machinery_info.cpp +++ b/src/machinery/machinery_info.cpp @@ -17,7 +17,7 @@ #include "dindexer-machinery/machinery_info.hpp" #include "dindexerConfig.h" -#include "helpers/stringize.h" +#include "duckhandy/stringize.h" namespace mchlib { boost::string_ref lib_signature() { diff --git a/src/machinery/scantask/dirtree.cpp b/src/machinery/scantask/dirtree.cpp index 4cd3764..875652a 100644 --- a/src/machinery/scantask/dirtree.cpp +++ b/src/machinery/scantask/dirtree.cpp @@ -18,7 +18,7 @@ #include "dindexer-machinery/scantask/dirtree.hpp" #include "dindexer-machinery/recorddata.hpp" #include "dindexer-machinery/set_listing.hpp" -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" #include "filesearcher.hpp" #include "pathname.hpp" #include diff --git a/src/machinery/scantask/mediatype.cpp b/src/machinery/scantask/mediatype.cpp index 1374c3f..ddd244a 100644 --- a/src/machinery/scantask/mediatype.cpp +++ b/src/machinery/scantask/mediatype.cpp @@ -21,6 +21,7 @@ # include "dindexer-machinery/mediatype.hpp" #endif #include "dindexer-machinery/recorddata.hpp" +#include "discinfo.hpp" #include namespace mchlib { @@ -57,6 +58,12 @@ namespace mchlib { #else data.type = media_type_to_char(m_default); #endif + + { + DiscInfo info((std::string(m_search_path))); + data.disk_label = info.label(); + data.fs_uuid = info.filesystem_uuid(); + } } } //namespace scantask } //namespace mchlib diff --git a/src/machinery/scantask/setbasic.cpp b/src/machinery/scantask/setbasic.cpp index 0538aed..53a5f97 100644 --- a/src/machinery/scantask/setbasic.cpp +++ b/src/machinery/scantask/setbasic.cpp @@ -29,13 +29,15 @@ namespace mchlib { } void SetBasic::on_data_destroy (SetRecordDataFull& parData) { - static_cast(parData).name.clear(); parData.name.clear(); + parData.fs_uuid.clear(); + parData.disk_label.clear(); } void SetBasic::on_data_create (SetRecordDataFull& parData) { parData.name = m_set_name; - static_cast(parData).name = parData.name; + parData.fs_uuid.clear(); + parData.disk_label.clear(); } } //namespace scantask } //namespace mchlib diff --git a/src/machinery/tiger.cpp b/src/machinery/tiger.cpp index 4531ccd..58ec3cc 100644 --- a/src/machinery/tiger.cpp +++ b/src/machinery/tiger.cpp @@ -16,6 +16,7 @@ */ #include "dindexer-machinery/tiger.hpp" +#include "duckhandy/lexical_cast.hpp" #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include extern "C" void tiger ( const char* parStr, uint64_t parLength, uint64_t parHash[3], char parPadding ); @@ -153,6 +155,20 @@ namespace mchlib { return oss.str(); } + TigerHash string_to_tiger (const std::string& parString) { + using boost::string_ref; + using TigerPartType = decltype(TigerHash::part_a); + + assert(parString.size() == sizeof(TigerHash) * 2); + + TigerHash retval; + const string_ref inp(parString); + retval.part_a = swap_long(dhandy::lexical_cast(inp.substr(0, sizeof(TigerPartType) * 2))); + retval.part_b = swap_long(dhandy::lexical_cast(inp.substr(sizeof(TigerPartType) * 2, sizeof(TigerPartType) * 2))); + retval.part_c = swap_long(dhandy::lexical_cast(inp.substr(sizeof(TigerPartType) * 4, sizeof(TigerPartType) * 2))); + return retval; + } + void tiger_data (const std::string& parData, TigerHash& parHash) { tiger (parData.data(), parData.size(), parHash.data, g_tiger_padding); } diff --git a/src/main/CMakeLists.txt b/src/main/CMakeLists.txt index 1b16abb..8f0f1d8 100644 --- a/src/main/CMakeLists.txt +++ b/src/main/CMakeLists.txt @@ -12,7 +12,6 @@ add_executable(${PROJECT_NAME} make_timestamp(${bare_name} timestamp.h.in) target_include_directories(${PROJECT_NAME} - PRIVATE ${CMAKE_SOURCE_DIR}/include PRIVATE ${CMAKE_SOURCE_DIR}/lib/pbl/pbl/src/src PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/src/main/damerau_levenshtein.h b/src/main/damerau_levenshtein.h index 1bbcf74..1bc898d 100644 --- a/src/main/damerau_levenshtein.h +++ b/src/main/damerau_levenshtein.h @@ -18,7 +18,7 @@ #ifndef id88738025C6B24BDEB604A5AE3C36EE8D #define id88738025C6B24BDEB604A5AE3C36EE8D -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" #if defined(__cplusplus) # include #else diff --git a/src/main/findactions.c b/src/main/findactions.c index a2790db..9da2d95 100644 --- a/src/main/findactions.c +++ b/src/main/findactions.c @@ -17,7 +17,7 @@ #include "findactions.h" #include "dindexerConfig.h" -#include "helpers/lengthof.h" +#include "duckhandy/lengthof.h" #include #include #include diff --git a/src/main/main.c b/src/main/main.c index f961f0b..ed972b4 100644 --- a/src/main/main.c +++ b/src/main/main.c @@ -19,7 +19,7 @@ #include "dindexerConfig.h" #include "findactions.h" -#include "helpers/lengthof.h" +#include "duckhandy/lengthof.h" #include "builtin_feats.h" #include "damerau_levenshtein.h" #include diff --git a/src/main/timestamp.h.in b/src/main/timestamp.h.in index 22a9339..f2303df 100644 --- a/src/main/timestamp.h.in +++ b/src/main/timestamp.h.in @@ -18,7 +18,7 @@ #ifndef id4A9E94ACFA43498097637025A5BA69E5 #define id4A9E94ACFA43498097637025A5BA69E5 -#include "dindexer-common/cmake_on_off.h" +#include "duckhandy/cmake_on_off.h" #if CMAKE_@DINDEXER_WITH_BUILD_DATE@ # define DINDEXER_BUILD_DATE "@DINDEXER_BUILD_DATE@" diff --git a/src/main/utf8_ops.h b/src/main/utf8_ops.h index 9d9be43..e981bf9 100644 --- a/src/main/utf8_ops.h +++ b/src/main/utf8_ops.h @@ -20,7 +20,7 @@ #include #include -#include "helpers/compatibility.h" +#include "duckhandy/compatibility.h" typedef uint32_t Character; diff --git a/src/navigate/AutomemBase.hpp b/src/navigate/AutomemBase.hpp deleted file mode 100644 index fd2d78c..0000000 --- a/src/navigate/AutomemBase.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef idDB299AE3079F4830BB1D543C07F7B594 -#define idDB299AE3079F4830BB1D543C07F7B594 - -//Classes in this file are low level and only intended to be used by library -//code. They don't do much by themselves and shoud never be used from outside. - -#include -#include -#include -#include - -#if !defined(NDEBUG) -# define ASSERTIONSENABLED -#endif - -namespace din { - const size_t MAX_STACK_ALLOC_SIZE = 128; - - ///------------------------------------------------------------------------- - ///Heap-based allocation, only gets raw memory - ///------------------------------------------------------------------------- - template > - class AutomemRawBase_heap { - protected: - AutomemRawBase_heap ( void ); - AutomemRawBase_heap ( const AutomemRawBase_heap& ) = delete; //Copy ctor can't be implemented at this level - AutomemRawBase_heap ( AutomemRawBase_heap&& parOther ); - - template - T* GetNewT ( size_t parIndex, Args&&... parArgs ); - T* AllocMemory ( void ); - void FreeMemory ( void ) noexcept; - void swap ( AutomemRawBase_heap& parOther ) noexcept { std::swap(m_localMem, parOther.m_localMem); } - private: - void operator= (const AutomemRawBase_heap&); - -#if defined(ASSERTIONSENABLED) - typedef uintptr_t PTR_INT_TYPE; - static_assert(sizeof(PTR_INT_TYPE) == sizeof(char*), "Wrong uintptr_t size"); -#endif - T* m_localMem; - }; - - ///------------------------------------------------------------------------- - ///Stack-based allocation, only gets raw memory - ///------------------------------------------------------------------------- - template - class AutomemRawBase_stack { - protected: - AutomemRawBase_stack ( void ) = default; - AutomemRawBase_stack ( const AutomemRawBase_stack& ) = delete; //Copy ctor can't be implemented at this level - AutomemRawBase_stack ( AutomemRawBase_stack&& ) = delete; - ~AutomemRawBase_stack ( void ) = default; - - template - T* GetNewT ( size_t parIndex, Args&&... parArgs ); - T* AllocMemory ( void ); - void FreeMemory ( void ) noexcept { return; } - private: -#if defined(ASSERTIONSENABLED) - typedef uintptr_t PTR_INT_TYPE; - static_assert(sizeof(PTR_INT_TYPE) == sizeof(char*), "Wrong uintptr_t size"); -#endif - void operator= (const AutomemRawBase_stack&); - typename std::aligned_storage::type m_localMem[S]; - }; -} //namespace din - -#include "AutomemBase.inl" - -#if defined(ASSERTIONSENABLED) -# undef ASSERTIONSENABLED -#endif -#endif diff --git a/src/navigate/AutomemBase.inl b/src/navigate/AutomemBase.inl deleted file mode 100644 index ba25d2e..0000000 --- a/src/navigate/AutomemBase.inl +++ /dev/null @@ -1,118 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -namespace din { - namespace { -#if defined(ASSERTIONSENABLED) - const char g_guard = 0xAB; -#endif - } //unnamed namespace - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - AutomemRawBase_heap::AutomemRawBase_heap() { -#if !defined(NDEBUG) - m_localMem = nullptr; -#endif - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - AutomemRawBase_heap::AutomemRawBase_heap (AutomemRawBase_heap&& parOther) { -#if !defined(NDEBUG) - m_localMem = nullptr; -#endif - this->swap(parOther); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - T* AutomemRawBase_heap::AllocMemory() { -#if !defined(NDEBUG) - Assert(nullptr == m_localMem); -#endif - m_localMem = A().allocate(S); -#if defined(ASSERTIONSENABLED) - assert(reinterpret_cast(m_localMem) % alignof(T) == 0); //Make sure alignment is correct - std::fill( - reinterpret_cast(&m_localMem[0]), - reinterpret_cast(&m_localMem[0]) + sizeof(m_localMem), - g_guard - ); -#endif - return m_localMem; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void AutomemRawBase_heap::FreeMemory() noexcept { -#if !defined(NDEBUG) - Assert(nullptr != m_localMem); -#endif - A().deallocate(m_localMem, S); -#if !defined(NDEBUG) - m_localMem = nullptr; -#endif - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - template - T* AutomemRawBase_heap::GetNewT (size_t parIndex, Args&&... parArgs) { - assert(parIndex < S); - T* const location = m_localMem + parIndex; -#if defined(ASSERTIONSENABLED) - assert(reinterpret_cast(location) % alignof(T) == 0); - assert(g_guard == *reinterpret_cast(location)); -#endif - return new(location) T(std::forward(parArgs)...); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - T* AutomemRawBase_stack::AllocMemory() { -#if defined(ASSERTIONSENABLED) - assert(reinterpret_cast(m_localMem) % alignof(T) == 0); //Make sure alignment is correct - std::fill( - reinterpret_cast(&m_localMem[0]), - reinterpret_cast(&m_localMem[0]) + sizeof(m_localMem), - g_guard - ); -#endif - return reinterpret_cast(&m_localMem[0]); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - template - T* AutomemRawBase_stack::GetNewT (size_t parIndex, Args&&... parArgs) { - assert(parIndex < S); - auto* const location = &m_localMem[parIndex]; -#if defined(ASSERTIONSENABLED) - assert(reinterpret_cast(location) % alignof(T) == 0); - assert(g_guard == *reinterpret_cast(location)); -#endif - return new(location) T(std::forward(parArgs)...); - } -} //namespace din diff --git a/src/navigate/CMakeLists.txt b/src/navigate/CMakeLists.txt index 86c0986..6e8cf88 100644 --- a/src/navigate/CMakeLists.txt +++ b/src/navigate/CMakeLists.txt @@ -5,7 +5,6 @@ add_executable(${PROJECT_NAME} commandline.cpp commandprocessor.cpp entrypath.cpp - dbsource.cpp linereader.cpp listdircontent.cpp ) diff --git a/src/navigate/IteratorOnPtr.hpp b/src/navigate/IteratorOnPtr.hpp deleted file mode 100644 index c384c40..0000000 --- a/src/navigate/IteratorOnPtr.hpp +++ /dev/null @@ -1,314 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef id1B822B1775424D879AA6AD1739B5BC91 -#define id1B822B1775424D879AA6AD1739B5BC91 - -#include -#include -#include -#include - -namespace din { - namespace implem_iop { - template - class IteratorOnPtr_base : public std::iterator { - typedef std::iterator parent_type; - public: - //Typedefs to be STL-compliant - typedef typename parent_type::iterator_category iterator_category; - typedef typename parent_type::value_type value_type; - typedef typename parent_type::difference_type difference_type; - typedef typename parent_type::pointer pointer; - typedef typename std::remove_reference::type& reference; - - IteratorOnPtr_base ( P parPointer, difference_type parSize ); - IteratorOnPtr_base ( const IteratorOnPtr_base& parOther ); - template - explicit IteratorOnPtr_base ( const IteratorOnPtr_base& parOther ); - ~IteratorOnPtr_base ( void ); - - difference_type operator- ( const IteratorOnPtr_base& parOther ) const; - - bool operator== ( const IteratorOnPtr_base& parOther ) const { return m_pointer == parOther.m_pointer; } - bool operator!= ( const IteratorOnPtr_base& parOther ) const { return not operator==(parOther); } - bool operator< ( const IteratorOnPtr_base& parOther ) const { return m_pointer < parOther.m_pointer; } - bool operator>= ( const IteratorOnPtr_base& parOther ) const { return not operator<(parOther); } - bool operator> ( const IteratorOnPtr_base& parOther ) const { return parOther < *this; } - bool operator<= ( const IteratorOnPtr_base& parOther ) const { return not operator>(parOther); } - - P GetPointer ( void ) { return m_pointer; } - const P GetPointer ( void ) const { return m_pointer; } -#if !defined(NDEBUG) - difference_type GetSize ( void ) const { return GetSize(0); } -#endif - - protected: - enum { - STEP = (I < 0 ? -I : I) - }; - -#if !defined(NDEBUG) - difference_type GetSize ( difference_type parAdvance ) const; - bool CanAdvance ( difference_type parAdvance ) const; - void AlterSize ( difference_type parAdvance ); -#endif - void MoveIterator ( difference_type parAdvance ); - - P m_pointer; - - private: -#if !defined(NDEBUG) - difference_type m_size; -#endif - }; - } //namespace implem_iop - - template - class IteratorOnPtr : public implem_iop::IteratorOnPtr_base { - typedef implem_iop::IteratorOnPtr_base parent_type; - enum { - STEP = parent_type::STEP - }; - public: - typedef typename parent_type::iterator_category iterator_category; - typedef typename parent_type::value_type value_type; - typedef typename parent_type::difference_type difference_type; - typedef typename parent_type::pointer pointer; - typedef typename parent_type::reference reference; - - IteratorOnPtr ( void ); - IteratorOnPtr ( P parPointer, difference_type parSize ); - IteratorOnPtr ( const IteratorOnPtr& parOther ) : parent_type(parOther) { return; } - template - IteratorOnPtr ( const IteratorOnPtr& parOther ) : parent_type(parOther) {} - ~IteratorOnPtr ( void ); - - IteratorOnPtr& operator++ ( void ); //pre - IteratorOnPtr operator++ ( int ); //post - IteratorOnPtr& operator-- ( void ); - IteratorOnPtr operator-- ( int ); - reference operator* ( void ); - pointer operator-> ( void ); - - using parent_type::operator-; - IteratorOnPtr operator+ ( difference_type parOther ) const; - IteratorOnPtr operator- ( difference_type parOther ) const; - IteratorOnPtr& operator+= ( difference_type parOther ); - IteratorOnPtr& operator-= ( difference_type parOther ); - protected: - private: - }; - - namespace implem_iop { - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template -#if !defined(NDEBUG) - IteratorOnPtr_base::IteratorOnPtr_base (P parPointer, difference_type parSize) : - m_pointer(parPointer), - m_size(parSize) -#else - IteratorOnPtr_base::IteratorOnPtr_base (P parPointer, difference_type) : - m_pointer(parPointer) -#endif - { - static_assert(I != 0, "Step must be non-zero"); - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - template - IteratorOnPtr_base::IteratorOnPtr_base (const IteratorOnPtr_base& parOther) { - m_pointer = parOther.GetPointer(); -#if !defined(NDEBUG) - m_size = parOther.GetSize(); -#endif - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - IteratorOnPtr_base::IteratorOnPtr_base (const IteratorOnPtr_base& parOther) { - m_pointer = parOther.m_pointer; -#if !defined(NDEBUG) - m_size = parOther.m_size; -#endif - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - IteratorOnPtr_base::~IteratorOnPtr_base() { - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - typename IteratorOnPtr_base::difference_type IteratorOnPtr_base::operator- (const IteratorOnPtr_base& parOther) const { - if (I > 0) - return m_pointer - parOther.m_pointer; - else - return parOther.m_pointer - m_pointer; - } - -#if !defined(NDEBUG) - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - typename IteratorOnPtr_base::difference_type IteratorOnPtr_base::GetSize (difference_type parAdvance) const { - return m_size - STEP * parAdvance; - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - bool IteratorOnPtr_base::CanAdvance (difference_type parAdvance) const { - return (m_size >= STEP * parAdvance); - } - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - void IteratorOnPtr_base::AlterSize (difference_type parAdvance) { - m_size = GetSize(parAdvance); - } -#endif - - ///--------------------------------------------------------------------- - ///--------------------------------------------------------------------- - template - void IteratorOnPtr_base::MoveIterator (difference_type parAdvance) { -#if !defined(NDEBUG) - assert(CanAdvance(parAdvance)); - AlterSize(parAdvance); -#endif - m_pointer += I * parAdvance; - } - } //namespace implem_iop - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr::IteratorOnPtr() : - parent_type(NULL, 0) - { - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr::IteratorOnPtr (P parPointer, difference_type parSize) : - parent_type(parPointer, parSize) - { - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr::~IteratorOnPtr() { - } - - ///------------------------------------------------------------------------- - ///Pre-increment. - ///------------------------------------------------------------------------- - template - IteratorOnPtr& IteratorOnPtr::operator++() { - this->MoveIterator(1); - return *this; - } - - ///------------------------------------------------------------------------- - ///Post-increment. - ///------------------------------------------------------------------------- - template - IteratorOnPtr IteratorOnPtr::operator++ (int) { - IteratorOnPtr retVal(*this); - this->MoveIterator(1); - return retVal; - } - - ///------------------------------------------------------------------------- - ///Pre-decrement. - ///------------------------------------------------------------------------- - template - IteratorOnPtr& IteratorOnPtr::operator--() { - this->MoveIterator(-1); - return *this; - } - - ///------------------------------------------------------------------------- - ///Post-decrement. - ///------------------------------------------------------------------------- - template - IteratorOnPtr IteratorOnPtr::operator-- (int) { - IteratorOnPtr retVal(*this); - this->MoveIterator(-1); - return retVal; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename IteratorOnPtr::reference IteratorOnPtr::operator*() { - return *(this->m_pointer); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename IteratorOnPtr::pointer IteratorOnPtr::operator->() { - return this->m_pointer; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr IteratorOnPtr::operator+ (difference_type parOther) const { - IteratorOnPtr retVal(*this); - retVal += parOther; - return retVal; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr IteratorOnPtr::operator- (difference_type parOther) const { - IteratorOnPtr retVal(*this); - retVal -= parOther; - return retVal; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr& IteratorOnPtr::operator+= (difference_type parOther) { - this->MoveIterator(parOther); - return *this; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - IteratorOnPtr& IteratorOnPtr::operator-= (difference_type parOther) { - this->MoveIterator(-parOther); - return *this; - } -} //namespace din - -#endif diff --git a/src/navigate/MaxSizedArray.hpp b/src/navigate/MaxSizedArray.hpp deleted file mode 100644 index 6082b7b..0000000 --- a/src/navigate/MaxSizedArray.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef idBC9F804ADD33468A9C7657E823FFC706 -#define idBC9F804ADD33468A9C7657E823FFC706 - -#include "AutomemBase.hpp" -#include "IteratorOnPtr.hpp" -#include -#include -#include -#include -#include - -namespace din { - const size_t MAXSZARR_MAX_STACK_ALLOC_SIZE = MAX_STACK_ALLOC_SIZE; - - //TODO: add a template parameter to force stack allocation regardless of - //sizeof(T) * S - template > - class MaxSizedArray : - private std::conditional< - (sizeof(T) * S > (MAXSZARR_MAX_STACK_ALLOC_SIZE > 4 * sizeof(T*) ? MAXSZARR_MAX_STACK_ALLOC_SIZE - sizeof(T*) : MAXSZARR_MAX_STACK_ALLOC_SIZE)), - AutomemRawBase_heap, - AutomemRawBase_stack - >::type - { - typedef typename std::conditional< - (sizeof(T) * S > (MAXSZARR_MAX_STACK_ALLOC_SIZE > 4 * sizeof(T*) ? MAXSZARR_MAX_STACK_ALLOC_SIZE - sizeof(T*) : MAXSZARR_MAX_STACK_ALLOC_SIZE)), - AutomemRawBase_heap, - AutomemRawBase_stack - >::type parent_type; - - public: - typedef IteratorOnPtr iterator; - typedef IteratorOnPtr const_iterator; - typedef IteratorOnPtr reverse_iterator; - typedef IteratorOnPtr const_reverse_iterator; - typedef T& reference; - typedef T&& mov_reference; - typedef const T& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T* pointer; - typedef const T* const_pointer; - - enum MAX_SIZE_ENUM { - MAX_SIZE = S - }; - - MaxSizedArray ( void ); - MaxSizedArray ( const MaxSizedArray& parOther ); - MaxSizedArray ( MaxSizedArray&& parOther ); - ~MaxSizedArray ( void ); - - bool empty ( void ) const; - size_type size ( void ) const; - size_type capacity ( void ) const { return S; } - size_type max_size ( void ) const { return S; } - void push_back ( const value_type& parNewItem ); - void push_back ( value_type&& parNewItem ); - void pop_back ( void ); - iterator erase ( const iterator& parDele ); - iterator erase ( const iterator& parFrom, const iterator& parToExcl ); - void clear ( void ); - mov_reference operator[] ( size_type parIndex ); - const_reference operator[] ( size_type parIndex ) const; - MaxSizedArray& operator= ( const MaxSizedArray& parOther ); - bool operator== ( const MaxSizedArray& parOther ) const; - bool operator!= ( const MaxSizedArray& parOther ) const; - pointer GetPointer ( void ); - const_pointer GetPointer ( void ) const; - void reserve ( size_type parReserve ); - - iterator begin ( void ); - const_iterator begin ( void ) const; - iterator end ( void ); - const_iterator end ( void ) const; - reverse_iterator rbegin ( void ); - const_reverse_iterator rbegin ( void ) const; - reverse_iterator rend ( void ); - const_reverse_iterator rend ( void ) const; - reference front ( void ) { return (*this)[0]; } - reference back ( void ) { return (*this)[size() - 1]; } - const_reference front ( void ) const { return (*this)[0]; } - const_reference back ( void ) const { return (*this)[size() - 1]; } - - private: - pointer GetPointer_NoAssert ( void ) { return m_localMem; } - const_pointer GetPointer_NoAssert ( void ) const { return m_localMem; } - - T* m_localMem; //A copy of memory pointer is required - size_type m_used; - }; -} //namespace din - -#include "MaxSizedArray.inl" - -#endif diff --git a/src/navigate/MaxSizedArray.inl b/src/navigate/MaxSizedArray.inl deleted file mode 100644 index e89f6a2..0000000 --- a/src/navigate/MaxSizedArray.inl +++ /dev/null @@ -1,281 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -namespace din { - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - MaxSizedArray::MaxSizedArray() { - m_localMem = this->AllocMemory(); - m_used = 0; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - MaxSizedArray::MaxSizedArray (const MaxSizedArray& parOther) : - parent_type(), - m_used(0) - { - m_localMem = this->AllocMemory(); - const size_type count = parOther.size(); - for (size_type z = 0; z < count; ++z) { - this->push_back(parOther[z]); - } - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - MaxSizedArray::MaxSizedArray (MaxSizedArray&& parOther) : - parent_type(), - m_used(0) - { - m_localMem = this->AllocMemory(); - const size_type count = parOther.size(); - for (size_type z = 0; z < count; ++z) { - this->push_back(std::move(parOther[z])); - } - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - MaxSizedArray::~MaxSizedArray() { - this->clear(); - this->FreeMemory(); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - bool MaxSizedArray::empty() const { - return 0 == m_used; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::size_type MaxSizedArray::size() const { - return m_used; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void MaxSizedArray::push_back (value_type&& parNewItem) { - assert(size() < capacity()); - this->GetNewT(m_used, std::move(parNewItem)); - ++m_used; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void MaxSizedArray::push_back (const value_type& parNewItem) { - assert(size() < capacity()); - this->GetNewT(m_used, parNewItem); - ++m_used; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void MaxSizedArray::pop_back() { - assert(not empty()); - m_localMem[m_used - 1].~T(); - --m_used; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::mov_reference MaxSizedArray::operator[] (size_type parIndex) { - assert(parIndex < size()); - return std::move(m_localMem[parIndex]); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_reference MaxSizedArray::operator[] (size_type parIndex) const { - assert(parIndex < size()); - return m_localMem[parIndex]; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - MaxSizedArray& MaxSizedArray::operator= (const MaxSizedArray& parOther) { - m_used = parOther.m_used; - std::copy(parOther.GetMemPtr(), parOther.GetMemPtr() + parOther.size(), this->GetMemPtr()); - return *this; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - bool MaxSizedArray::operator== (const MaxSizedArray& parOther) const { - if (size() != parOther.size()) - return false; - for (size_type z = 0; z < size(); ++z) { - if ((*this)[z] != parOther[z]) - return false; - } - return true; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - bool MaxSizedArray::operator!= (const MaxSizedArray& parOther) const { - return not (*this == parOther); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::pointer MaxSizedArray::GetPointer() { - assert(size() > 0); - return GetPointer_NoAssert(); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_pointer MaxSizedArray::GetPointer() const { - assert(size() > 0); - return GetPointer_NoAssert(); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void MaxSizedArray::clear() { - const size_type count = this->size(); - for (size_type z = 0; z < count; ++z) { - (*this)[z].~T(); - } - m_used = 0; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::iterator MaxSizedArray::begin() { - return iterator(GetPointer_NoAssert(), size()); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_iterator MaxSizedArray::begin() const { - return const_iterator(GetPointer_NoAssert(), size()); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::iterator MaxSizedArray::end() { - return iterator(GetPointer_NoAssert() + size(), 0); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_iterator MaxSizedArray::end() const { - return const_iterator(GetPointer_NoAssert() + size(), 0); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::reverse_iterator MaxSizedArray::rbegin() { - return reverse_iterator(GetPointer_NoAssert() + size() - 1, size()); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_reverse_iterator MaxSizedArray::rbegin() const { - return const_reverse_iterator(GetPointer_NoAssert() + size() - 1, size()); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::reverse_iterator MaxSizedArray::rend() { - return reverse_iterator(GetPointer_NoAssert() - 1, 0); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::const_reverse_iterator MaxSizedArray::rend() const { - return const_reverse_iterator(GetPointer_NoAssert() - 1, 0); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::iterator MaxSizedArray::erase (const iterator& parDele) { - assert(end() != parDele); - return erase(parDele, parDele + 1); - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - typename MaxSizedArray::iterator MaxSizedArray::erase (const iterator& parFrom, const iterator& parToExcl) { - assert(parFrom >= begin()); - assert(parToExcl <= end()); - assert(parToExcl >= parFrom); - - //I'm doing this in two passes: first, I'm deleting as many elements as - //the ones that would be left at the end (that is, delete and move), - //then delete everything to the end of the buffer, if necessary. - const size_type deleCount = static_cast(parToExcl - parFrom); - const size_type firstIndexToDele = static_cast(parFrom - begin()); - const size_type& sz = m_used; - assert(firstIndexToDele + deleCount <= sz); - const size_type deleAndCopyCount = sz - (firstIndexToDele + deleCount); - - //As said, make room and copy from the cut tail - for (size_type z = firstIndexToDele; z < firstIndexToDele + deleAndCopyCount; z++) { - (*this)[z].~T(); - new(&(*this)[z]) T((*this)[z + deleCount]); - } - - //Any leftover is rubbish - for (size_type z = firstIndexToDele + deleAndCopyCount; z < sz; z++) { - (*this)[z].~T(); - } - - m_used -= deleCount; - return begin() + firstIndexToDele; - } - - ///------------------------------------------------------------------------- - ///------------------------------------------------------------------------- - template - void MaxSizedArray::reserve (size_type parReserve) { - assert(parReserve <= S); - if (parReserve > S) { - throw std::length_error("Unable to reserve more memory than the build-time size for MaxSizedArray"); - } - } -} //namespace din diff --git a/src/navigate/dbsource.cpp b/src/navigate/dbsource.cpp deleted file mode 100644 index 02100f3..0000000 --- a/src/navigate/dbsource.cpp +++ /dev/null @@ -1,183 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#include "dbsource.hpp" -#include "dindexer-common/settings.hpp" -#include "pq/connection.hpp" -#include "helpers/infix_iterator.hpp" -#include -#include -#include -#include -#include - -namespace din { - namespace { - const uint32_t g_files_query_limit = 500; - - std::ostream& operator<< (std::ostream& parOut, const std::vector& parCols) { - parOut << '"'; - boost::copy(parCols, infix_ostream_iterator(parOut, "\", \"")); - parOut << '"'; - return parOut; - } - } //unnamed namespace - - const DBSource::SetDetailsMap DBSource::m_set_details_map { - {SetDetail_Desc, "desc"}, - {SetDetail_Type, "type"}, - {SetDetail_CreeationDate, "creation"}, - {SetDetail_AppName, "app_name"}, - {SetDetail_ID, "id"} - }; - const DBSource::FileDetailsMap DBSource::m_file_details_map { - {FileDetail_ID, "id"}, - {FileDetail_Path, "path"}, - {FileDetail_Level, "level"}, - {FileDetail_GroupID, "group_id"}, - {FileDetail_IsDir, "is_directory"}, - {FileDetail_IsSymLink, "is_symlink"}, - {FileDetail_Size, "size"}, - {FileDetail_Hash, "hash"}, - {FileDetail_IsHashValid, "is_hash_valid"}, - {FileDetail_ATime, "access_time"}, - {FileDetail_MTime, "modify_time"}, - {FileDetail_Unreadable, "unreadable"}, - {FileDetail_MimeType, "mimetype"}, - {FileDetail_Charset, "charset"} - }; - - struct DBSource::LocalData { - explicit LocalData ( const dinlib::SettingsDB& parDBSettings ) : - conn( - std::string(parDBSettings.username), - std::string(parDBSettings.password), - std::string(parDBSettings.dbname), - std::string(parDBSettings.address), - parDBSettings.port - ) - { - } - - pq::Connection conn; - }; - - DBSource::DBSource (const dinlib::SettingsDB& parDBSettings) : - m_local_data(new LocalData(parDBSettings)) - { - assert(not m_local_data->conn.is_connected()); - } - - DBSource::~DBSource() noexcept { - } - - void DBSource::disconnect() { - if (m_local_data->conn.is_connected()) { - m_local_data->conn.disconnect(); - } - } - - pq::Connection& DBSource::get_conn() { - if (not m_local_data->conn.is_connected()) { - m_local_data->conn.connect(); - } - return m_local_data->conn; - } - - std::vector DBSource::sets() { - using boost::lexical_cast; - - auto& conn = get_conn(); - const std::string query = "SELECT \"id\" FROM \"sets\";"; - auto res = conn.query(query); - std::vector retval; - - retval.reserve(res.size()); - for (const auto& row : res) { - retval.push_back(lexical_cast(row[0])); - } - return retval; - } - - void DBSource::query_no_conditions (const ColumnList& parCols, boost::string_ref parTable, const std::vector& parIDs, std::function parCallback) { - std::ostringstream oss; - oss << "SELECT " << parCols << ' ' << - "FROM \"" << parTable << "\" " << - "WHERE \"id\"=ANY($1) " << - "ORDER BY \"desc\" ASC " << - "LIMIT " << g_files_query_limit << ';'; - - auto& conn = get_conn(); - auto result = conn.query(oss.str(), parIDs); - for (auto row : result) { - for (auto val : row) { - parCallback(std::move(val)); - } - } - } - - void DBSource::query_files_in_dir (const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, uint32_t parGroupID, QueryCallback parCallback) { - std::ostringstream oss; - oss << "SELECT " << parCols << ' ' << - "FROM \"files\" WHERE " << - "\"level\"=$1 " << - "AND \"group_id\"=$2 " << - "AND str_begins_with(\"path\", COALESCE($3, '')) " << - "ORDER BY \"is_directory\" DESC, \"path\" ASC " << - "LIMIT " << g_files_query_limit << ';'; - - auto& conn = get_conn(); - auto result = conn.query( - oss.str(), - parLevel, - parGroupID, - parDir - ); - for (auto row : result) { - for (auto val : row) { - parCallback(std::move(val)); - } - } - } - - std::vector DBSource::paths_starting_by (uint32_t parGroupID, uint16_t parLevel, boost::string_ref parPath) { - std::ostringstream oss; - oss << "SELECT \"path\" ||\n" << - "(SELECT CASE \"is_directory\"\n" << - "WHEN TRUE THEN '/'\n" << - "ELSE ''\n" << - "END) as path FROM \"files\" WHERE \"group_id\"=$1 AND " << - "\"level\"=$2 AND str_begins_with(\"path\", COALESCE($3, '')) " << - "ORDER BY \"is_directory\" DESC, \"path\" ASC LIMIT " << - g_files_query_limit << ';'; - - auto& conn = get_conn(); - auto result = conn.query( - oss.str(), - parGroupID, - parLevel, - parPath - ); - std::vector retval; - retval.reserve(retval.size()); - for (auto row : result) { - assert(not row.empty()); - retval.push_back(row[0]); - } - return retval; - } -} //namespace din diff --git a/src/navigate/entrypath.cpp b/src/navigate/entrypath.cpp index 31b5e27..5eafdad 100644 --- a/src/navigate/entrypath.cpp +++ b/src/navigate/entrypath.cpp @@ -16,7 +16,7 @@ */ #include "entrypath.hpp" -#include "helpers/infix_iterator.hpp" +#include "duckhandy/infix_iterator.hpp" #include #include #include diff --git a/src/navigate/listdircontent.cpp b/src/navigate/listdircontent.cpp index fef051c..76f8b04 100644 --- a/src/navigate/listdircontent.cpp +++ b/src/navigate/listdircontent.cpp @@ -17,8 +17,8 @@ #include "listdircontent.hpp" #include "entrypath.hpp" -#include "dbsource.hpp" -#include "helpers/infix_iterator.hpp" +#include "backends/db_backend.hpp" +#include "duckhandy/infix_iterator.hpp" #include #include #include @@ -68,7 +68,7 @@ namespace din { } } //unnamed namespace - ListDirContent::ListDirContent (DBSource* parDB) : + ListDirContent::ListDirContent (dindb::Backend* parDB) : m_cache(g_max_cached_lists), m_db(parDB) { @@ -85,14 +85,14 @@ namespace din { //Requested item is not cached, so we need to query the db now if (parDir.points_to_group()) { - auto sets_ids = m_db->sets(); - auto sets_info = m_db->set_details(sets_ids); + auto sets_ids = m_db->find_all_sets(); + auto sets_info = m_db->find_set_details(sets_ids); m_cache.push_back(std::make_pair(curr_path, db_result_to_vec(sets_info))); } else { auto path_prefix = parDir.file_path(); const auto set_id = parDir.group_id(); - auto files_info = m_db->file_details(set_id, parDir.level() + 1, path_prefix); + auto files_info = m_db->find_file_details(set_id, parDir.level() + 1, path_prefix); m_cache.push_back(std::make_pair(curr_path, db_result_to_vec(files_info))); } return last_cached_item(curr_path); @@ -115,7 +115,7 @@ namespace din { else { const auto set_id = parDir.group_id(); const auto path_prefix = parDir.file_path(); - auto file_list = m_db->paths_starting_by(set_id, parDir.level(), path_prefix); + auto file_list = m_db->find_paths_starting_by(set_id, parDir.level(), path_prefix); for (auto& file_item : file_list) { file_item = EntryPath(file_item)[return_level]; } diff --git a/src/navigate/listdircontent.hpp b/src/navigate/listdircontent.hpp index 9a02e40..ad20f27 100644 --- a/src/navigate/listdircontent.hpp +++ b/src/navigate/listdircontent.hpp @@ -23,15 +23,18 @@ #include #include +namespace dindb { + class Backend; +} //namespace dindb + namespace din { class EntryPath; - class DBSource; class ListDirContent { using ListType = std::vector; using CachedItemType = std::pair; public: - explicit ListDirContent ( DBSource* parDB ); + explicit ListDirContent ( dindb::Backend* parDB ); ~ListDirContent ( void ) noexcept = default; const ListType& ls ( const EntryPath& parDir ) const; @@ -41,7 +44,7 @@ namespace din { const ListType& last_cached_item ( const std::string& parCurrPath ) const; mutable boost::circular_buffer m_cache; - DBSource* m_db; + dindb::Backend* m_db; }; } //namespace din diff --git a/src/navigate/main.cpp b/src/navigate/main.cpp index 594ba89..c0436d5 100644 --- a/src/navigate/main.cpp +++ b/src/navigate/main.cpp @@ -19,7 +19,6 @@ #include "commandprocessor.hpp" #include "dindexer-common/settings.hpp" #include "entrypath.hpp" -#include "dbsource.hpp" #include "dindexerConfig.h" #include "linereader.hpp" #include "listdircontent.hpp" @@ -27,11 +26,10 @@ #include #include #include -#include #include namespace { - void do_navigation ( din::DBSource& parDB ); + void do_navigation ( dindb::Backend& parDB ); bool on_exit ( void ); void on_pwd ( const din::EntryPath& parDirMan ); @@ -53,17 +51,16 @@ int main (int parArgc, char* parArgv[]) { } dinlib::Settings settings; - { - const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings); - if (not loaded) { - std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n"; - return 1; - } + try { + dinlib::load_settings(CONFIG_FILE_PATH, settings); + } + catch (const std::runtime_error& err) { + std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n"; + std::cerr << err.what() << '\n'; + return 1; } - din::DBSource db_source(settings.db); - - do_navigation(db_source); + do_navigation(settings.backend_plugin.backend()); return 0; } @@ -81,7 +78,7 @@ namespace { boost::copy(ls_result, std::ostream_iterator(std::cout, "\n")); } - void do_navigation (din::DBSource& parDB) { + void do_navigation (dindb::Backend& parDB) { const std::string prompt; din::ListDirContent ls(&parDB); din::LineReader lines(&ls); @@ -92,7 +89,7 @@ namespace { din::EntryPath dir_man; proc.add_command("exit", &on_exit, 0); proc.add_command("cd", std::function(std::bind(&din::EntryPath::push_piece, &dir_man, std::placeholders::_1)), 1); - proc.add_command("disconnect", std::function(std::bind(&din::DBSource::disconnect, &parDB)), 0); + proc.add_command("disconnect", std::function(std::bind(&dindb::Backend::disconnect, std::ref(parDB))), 0); proc.add_command("pwd", std::function(std::bind(&on_pwd, std::ref(dir_man))), 0); proc.add_command("ls", std::function(std::bind(on_ls, std::ref(ls), std::ref(dir_man))), 0); do { diff --git a/src/pq/CMakeLists.txt b/src/pq/CMakeLists.txt index 543e6e3..d164d80 100644 --- a/src/pq/CMakeLists.txt +++ b/src/pq/CMakeLists.txt @@ -1,5 +1,8 @@ project(${bare_name}-pq CXX) +set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fPIC") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fPIC") + add_library(${PROJECT_NAME} STATIC connection.cpp databaseexception.cpp @@ -9,10 +12,12 @@ add_library(${PROJECT_NAME} STATIC target_include_directories(${PROJECT_NAME} PRIVATE ${libpqtypes_INCLUDE_DIRS} + PRIVATE ${PostgreSQL_INCLUDE_DIRS} ) target_link_libraries(${PROJECT_NAME} PRIVATE ${bare_name}-if PRIVATE pqtypes + PRIVATE ${PostgreSQL_LIBRARIES} ) target_compile_features(${PROJECT_NAME} diff --git a/src/scan/CMakeLists.txt b/src/scan/CMakeLists.txt index 7137249..0ac7189 100644 --- a/src/scan/CMakeLists.txt +++ b/src/scan/CMakeLists.txt @@ -4,7 +4,6 @@ include(WithMediaAutodetect) add_executable(${PROJECT_NAME} main.cpp - dbbackend.cpp commandline.cpp ) diff --git a/src/scan/main.cpp b/src/scan/main.cpp index a8c9cac..207feec 100644 --- a/src/scan/main.cpp +++ b/src/scan/main.cpp @@ -25,7 +25,6 @@ #include "dindexer-common/common_info.hpp" #include "dindexer-common/settings.hpp" #include "commandline.hpp" -#include "dbbackend.hpp" #include "dindexer-machinery/scantask/dirtree.hpp" #include "dindexer-machinery/scantask/mediatype.hpp" #include "dindexer-machinery/scantask/hashing.hpp" @@ -41,7 +40,7 @@ #endif namespace { - bool add_to_db ( const std::vector& parData, const mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDBSettings, bool parForce=false ); + bool add_to_db ( const std::vector& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce=false ); #if defined(WITH_PROGRESS_FEEDBACK) void print_progress ( const boost::string_ref parPath, uint64_t parFileBytes, uint64_t parTotalBytes, uint32_t parFileNum, std::size_t& parClearCount ); #endif @@ -74,12 +73,13 @@ int main (int parArgc, char* parArgv[]) { #endif dinlib::Settings settings; - { - const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings); - if (not loaded) { - std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n"; - return 1; - } + try { + dinlib::load_settings(CONFIG_FILE_PATH, settings); + } + catch (const std::runtime_error& err) { + std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n"; + std::cerr << err.what() << '\n'; + return 1; } bool ignore_read_errors = (vm.count("ignore-errors") > 0); @@ -104,7 +104,7 @@ int main (int parArgc, char* parArgv[]) { ); #endif - const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.db); + const bool added_to_db = add_to_db(filerecdata->get_or_create(), setrecdata->get_or_create(), settings.backend_plugin.backend()); #if defined(WITH_PROGRESS_FEEDBACK) std::cout << '\n'; #endif @@ -115,26 +115,24 @@ int main (int parArgc, char* parArgv[]) { } namespace { - bool add_to_db (const std::vector& parData, const mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDBSettings, bool parForce) { + bool add_to_db (const std::vector& parData, const mchlib::SetRecordDataFull& parSet, dindb::Backend& parDB, bool parForce) { using mchlib::FileRecordData; using mchlib::SetRecordDataFull; - using mchlib::SetRecordData; if (not parForce) { const auto& first_hash = parData.front().hash; FileRecordData itm; SetRecordDataFull set; - const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash); + const bool already_in_db = parDB.search_file_by_hash(itm, set, first_hash); if (already_in_db) { return false; } } - const SetRecordData& set_data {parSet.name, parSet.type, parSet.content_type }; const auto app_signature = dinlib::dindexer_signature(); const auto lib_signature = mchlib::lib_signature(); const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size()); - din::write_to_db(parDBSettings, parData, set_data, signature); + parDB.write_files(parData, parSet, signature); return true; } diff --git a/src/tag/CMakeLists.txt b/src/tag/CMakeLists.txt index e642dc2..89d5ec9 100644 --- a/src/tag/CMakeLists.txt +++ b/src/tag/CMakeLists.txt @@ -3,7 +3,6 @@ project(${bare_name}-tag CXX) add_executable(${PROJECT_NAME} main.cpp commandline.cpp - tag_postgres.cpp ) target_include_directories(${PROJECT_NAME} diff --git a/src/tag/main.cpp b/src/tag/main.cpp index ed0e7e1..f934d9a 100644 --- a/src/tag/main.cpp +++ b/src/tag/main.cpp @@ -18,8 +18,7 @@ #include "commandline.hpp" #include "dindexer-common/settings.hpp" #include "dindexerConfig.h" -#include "tag_postgres.hpp" -#include "dindexer-common/split_tags.hpp" +#include "dindexer-core/split_tags.hpp" #include "glob2regex/glob2regex.hpp" #include "enum.h" #include @@ -47,40 +46,34 @@ namespace { return retval; } - din::OwnerSetInfo make_owner_set_info (const boost::program_options::variables_map& parVM) { - din::OwnerSetInfo set_info; - if (parVM.count("set")) { - set_info.is_valid = true; - set_info.group_id = parVM["set"].as(); - } - else { - set_info.is_valid = false; - set_info.group_id = 0; - } - return set_info; + dindb::GroupIDType make_owner_set_info (const boost::program_options::variables_map& parVM) { + if (parVM.count("set")) + return parVM["set"].as(); + else + return dindb::InvalidGroupID; } - int tag_files (const dinlib::SettingsDB& parDB, TaggingMode parMode, const boost::program_options::variables_map& parVM, const std::vector& parTags) { + int tag_files (dindb::Backend& parDB, TaggingMode parMode, const boost::program_options::variables_map& parVM, const std::vector& parTags) { using boost::lexical_cast; using boost::string_ref; - const din::OwnerSetInfo set_info = make_owner_set_info(parVM); + const auto set_info = make_owner_set_info(parVM); switch (parMode) { case TaggingMode::ID: { - auto ids_string = dinlib::split_tags(parVM["ids"].as()); + auto ids_string = dincore::split_tags(parVM["ids"].as()); std::vector ids; ids.reserve(ids_string.size()); std::transform(ids_string.begin(), ids_string.end(), std::back_inserter(ids), &lexical_cast); - din::tag_files(parDB, ids, parTags, set_info); + parDB.tag_files(ids, parTags, set_info); return 0; } case TaggingMode::Glob: { const auto regexes(globs_to_regex_list(parVM["globs"].as>())); - din::tag_files(parDB, regexes, parTags, set_info); + parDB.tag_files(regexes, parTags, set_info); return 0; } @@ -90,7 +83,7 @@ namespace { } } - int delete_tags (const dinlib::SettingsDB& parDB, TaggingMode parMode, const boost::program_options::variables_map& parVM, const std::vector& parTags) { + int delete_tags (dindb::Backend& parDB, TaggingMode parMode, const boost::program_options::variables_map& parVM, const std::vector& parTags) { using boost::lexical_cast; using boost::string_ref; @@ -99,14 +92,14 @@ namespace { switch (parMode) { case TaggingMode::ID: { - auto ids_string = dinlib::split_tags(parVM["ids"].as()); + auto ids_string = dincore::split_tags(parVM["ids"].as()); std::vector ids; ids.reserve(ids_string.size()); std::transform(ids_string.begin(), ids_string.end(), std::back_inserter(ids), &lexical_cast); if (parVM.count("alltags")) - din::delete_all_tags(parDB, ids, make_owner_set_info(parVM)); + parDB.delete_all_tags(ids, make_owner_set_info(parVM)); else - din::delete_tags(parDB, ids, parTags, make_owner_set_info(parVM)); + parDB.delete_tags(ids, parTags, make_owner_set_info(parVM)); return 0; } @@ -114,9 +107,9 @@ namespace { { const auto regexes(globs_to_regex_list(parVM["globs"].as>())); if (parVM.count("alltags")) - din::delete_all_tags(parDB, regexes, make_owner_set_info(parVM)); + parDB.delete_all_tags(regexes, make_owner_set_info(parVM)); else - din::delete_tags(parDB, regexes, parTags, make_owner_set_info(parVM)); + parDB.delete_tags(regexes, parTags, make_owner_set_info(parVM)); return 0; } @@ -154,20 +147,21 @@ int main (int parArgc, char* parArgv[]) { assert(id_mode xor glob_mode); dinlib::Settings settings; - { - const bool loaded = dinlib::load_settings(CONFIG_FILE_PATH, settings); - if (not loaded) { - std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ", quitting\n"; - return 1; - } + try { + dinlib::load_settings(CONFIG_FILE_PATH, settings); + } + catch (const std::runtime_error& err) { + std::cerr << "Can't load settings from " << CONFIG_FILE_PATH << ":\n"; + std::cerr << err.what() << '\n'; + return 1; } const auto master_tags_string = vm["tags"].as(); - const std::vector tags = dinlib::split_tags(master_tags_string); + const std::vector tags = dincore::split_tags(master_tags_string); const auto mode = (glob_mode ? TaggingMode::Glob : TaggingMode::ID); if (not vm.count("delete")) - return tag_files(settings.db, mode, vm, tags); + return tag_files(settings.backend_plugin.backend(), mode, vm, tags); else - return delete_tags(settings.db, mode, vm, tags); + return delete_tags(settings.backend_plugin.backend(), mode, vm, tags); } diff --git a/src/tag/tag_postgres.hpp b/src/tag/tag_postgres.hpp deleted file mode 100644 index 5fd2608..0000000 --- a/src/tag/tag_postgres.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright 2015, 2016, Michele Santullo - * This file is part of "dindexer". - * - * "dindexer" is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * "dindexer" is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with "dindexer". If not, see . - */ - -#ifndef idE1E1650A8CAA4949BD6D4D58BF2599F5 -#define idE1E1650A8CAA4949BD6D4D58BF2599F5 - -#include -#include -#include - -namespace dinlib { - struct SettingsDB; -} //namespace dinlib - -namespace din { - struct OwnerSetInfo { - uint32_t group_id; - bool is_valid; - }; - - void tag_files ( const dinlib::SettingsDB& parDB, const std::vector& parFiles, const std::vector& parTags, OwnerSetInfo parSet ); - void tag_files ( const dinlib::SettingsDB& parDB, const std::vector& parRegexes, const std::vector& parTags, OwnerSetInfo parSet ); - - void delete_tags ( const dinlib::SettingsDB& parDB, const std::vector& parFiles, const std::vector& parTags, OwnerSetInfo parSet ); - void delete_tags ( const dinlib::SettingsDB& parDB, const std::vector& parRegexes, const std::vector& parTags, OwnerSetInfo parSet ); - void delete_all_tags ( const dinlib::SettingsDB& parDB, const std::vector& parFiles, OwnerSetInfo parSet ); - void delete_all_tags ( const dinlib::SettingsDB& parDB, const std::vector& parRegexes, OwnerSetInfo parSet ); -} //namespace din - -#endif diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index dfbe920..f2b4b94 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -4,6 +4,8 @@ add_executable(${PROJECT_NAME} test_diriterator.cpp test_guess_content_type.cpp test_glob2regex.cpp + test_tiger_string_conv.cpp + test_lexical_cast.cpp ) target_include_directories(${PROJECT_NAME} SYSTEM diff --git a/test/unit/test_diriterator.cpp b/test/unit/test_diriterator.cpp index 1cbcfb6..c57847d 100644 --- a/test/unit/test_diriterator.cpp +++ b/test/unit/test_diriterator.cpp @@ -17,7 +17,7 @@ #include "dindexer-machinery/set_listing.hpp" #include "dindexer-machinery/set_listing_helpers.hpp" -#include "helpers/lengthof.h" +#include "duckhandy/lengthof.h" #include #include #include diff --git a/test/unit/test_lexical_cast.cpp b/test/unit/test_lexical_cast.cpp new file mode 100644 index 0000000..f2aec65 --- /dev/null +++ b/test/unit/test_lexical_cast.cpp @@ -0,0 +1,165 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "duckhandy/lexical_cast.hpp" +#include +#include +#include +#include + +TEST(duckhandy, lexical_cast) { + using dhandy::lexical_cast; + using dhandy::tags::hex; + using dhandy::tags::dec; + using dhandy::tags::bin; + + { + const uint8_t value = 0x34; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("34", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("52", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("110100", bin_str); + } + { + const uint16_t value = 0xFFFF; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("FFFF", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("65535", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1111111111111111", bin_str); + } + { + const uint16_t value = 0xFF; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("FF", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("255", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("11111111", bin_str); + } + { + const uint16_t value = 0x100; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("100", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("256", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("100000000", bin_str); + } + { + const uint32_t value = 0x1ABC; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("1ABC", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("6844", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1101010111100", bin_str); + } + { + const uint32_t value = 0xffffffff; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("FFFFFFFF", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("4294967295", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("11111111111111111111111111111111", bin_str); + } + { + const uint32_t value = 1; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("1", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("1", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1", bin_str); + } + { + const uint32_t value = 0; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("0", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("0", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("0", bin_str); + } + { + const uint64_t value = 0x12abcd34; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("12ABCD34", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("313249076", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("10010101010111100110100110100", bin_str); + } + { + const int32_t value = 0x7800; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("7800", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("30720", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("111100000000000", bin_str); + } + { + const int32_t value = -1400; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("FFFFFA88", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("-1400", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("11111111111111111111101010001000", bin_str); + } + { + const int32_t value = 515; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("203", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("515", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1000000011", bin_str); + } + { + const int64_t value = 515; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("203", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("515", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1000000011", bin_str); + } + { + const uint32_t value = 515; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("203", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("515", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1000000011", bin_str); + } + { + const uint64_t value = 515; + const auto hex_str = lexical_cast(value); + EXPECT_EQ("203", hex_str); + const auto dec_str = lexical_cast(value); + EXPECT_EQ("515", dec_str); + const auto bin_str = lexical_cast(value); + EXPECT_EQ("1000000011", bin_str); + } +} diff --git a/test/unit/test_tiger_string_conv.cpp b/test/unit/test_tiger_string_conv.cpp new file mode 100644 index 0000000..2e04395 --- /dev/null +++ b/test/unit/test_tiger_string_conv.cpp @@ -0,0 +1,63 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "dindexer-machinery/tiger.hpp" +#include +#include + +TEST(machinery, TigerStringConv) { + using mchlib::TigerHash; + + const std::size_t TigerStringCharCount = sizeof(TigerHash) * 2; + { + TigerHash tiger_zero; + tiger_zero.part_a = tiger_zero.part_b = tiger_zero.part_c = 0; + + const std::string expected(TigerStringCharCount, '0'); + const std::string got = mchlib::tiger_to_string(tiger_zero); + EXPECT_EQ(expected, got); + + auto hash2 = mchlib::string_to_tiger(got); + EXPECT_EQ(tiger_zero, hash2); + } + + { + TigerHash t; + mchlib::tiger_init_hash(t); + + const std::string expected = "efcdab89674523011032547698badcfe87e1b2c3b4a596f0"; + EXPECT_EQ(TigerStringCharCount, expected.size()); + const std::string got = mchlib::tiger_to_string(t); + EXPECT_EQ(expected, got); + + auto hash2 = mchlib::string_to_tiger(got); + EXPECT_EQ(t, hash2); + } + + { + TigerHash t; + t.part_a = t.part_b = t.part_c = 0xababababababababULL; + + const std::string expected = "abababababababababababababababababababababababab"; + EXPECT_EQ(TigerStringCharCount, expected.size()); + const std::string got = mchlib::tiger_to_string(t); + EXPECT_EQ(expected, got); + + auto hash2 = mchlib::string_to_tiger(got); + EXPECT_EQ(t, hash2); + } +} diff --git a/test/unit_cli/test_tag_splitting.cpp b/test/unit_cli/test_tag_splitting.cpp index 08066f6..0b8ef5b 100644 --- a/test/unit_cli/test_tag_splitting.cpp +++ b/test/unit_cli/test_tag_splitting.cpp @@ -18,10 +18,10 @@ #include #include #include -#include "dindexer-common/split_tags.hpp" +#include "dindexer-core/split_tags.hpp" TEST(cli_main, split_tags) { - using dinlib::split_tags; + using dincore::split_tags; { const std::string test_tags = "tag1,tag2"; diff --git a/tools/flat_git.rb b/tools/flat_git.rb new file mode 100755 index 0000000..6f1e2f9 --- /dev/null +++ b/tools/flat_git.rb @@ -0,0 +1,179 @@ +#!/usr/bin/env ruby + +=begin +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to +=end + +require 'shellwords' +require 'pathname' +require 'yaml' + +GIT="git" + +SubmoduleInfo = Struct.new(:path, :url, :status, :name, :hash) + +def call_git(parTokens, parCaptureStdout=true, work_dir: nil) + git_c_param = "" + if work_dir then + raise ScriptError, "Tainted path received in call_git() function" if work_dir.tainted? + git_c_param = " -C #{work_dir}" + end + + command_line = "" + if parTokens.is_a?String then + raise ScriptError, "Tainted token received in call_git() function" if parTokens.tainted? + command_line = parTokens + elsif parTokens.is_a?Array then + parTokens.each do |tok,idx| + raise ScriptError, "Tainted token at index #{idx} received in call_git() function" if tok.tainted? + end + command_line = parTokens.join(" ") + else + raise ScriptError, "Invalid parTokens received in call_git() function" + end + + if parCaptureStdout then + return `#{GIT}#{git_c_param} #{command_line}`.chomp + else + return Kernel.system("#{GIT}#{git_c_param} #{command_line}") + end +end + +def is_absolute_url?(parUrl) + return not(/^\.{1,2}\// =~ parUrl) +end + +def sanitized(parString) + retval = Shellwords.escape(parString) + retval.untaint + return retval +end + +def current_remote_url() + current_branch = call_git(["rev-parse", "--abbrev-ref", "HEAD"]) + current_remote = call_git(["config", "branch.#{sanitized(current_branch)}.remote"]) + remote_url = call_git(["config", "remote.#{sanitized(current_remote)}.url"]) + return remote_url +end + +def submodules_info(parBaseRemoteUrl) + regex_submodule = /^submodule\.(.+?)\.([^.]+)$/ + submodules = Hash.new{ |hash, key| hash[key] = SubmoduleInfo.new } + call_git(["config", "--file", ".gitmodules", "--name-only", "--list"]).lines.map(&:chomp).each do |line| + m = regex_submodule.match(line) + next if m.nil? + + new_value = call_git(["config", "--file", ".gitmodules", sanitized(line)]) + case m[2] + when "url" then + submodules[m[1]].url = is_absolute_url?(new_value) ? new_value : File.join(parBaseRemoteUrl, new_value) + submodules[m[1]].name = File.basename(new_value, ".git") + when "path" then + submodules[m[1]].path = new_value + end + end + + regex_status = /^(.)([0-9a-fA-F]{40})\s+(.+?)(?:\s+\(.+\))?$/ + call_git(["submodule", "status"]).lines.map(&:chomp).each do |line| + m = regex_status.match(line) + next if m.nil? + + submodules[m[3]].status = m[1] + submodules[m[3]].hash = m[2] + end + return submodules +end + +def is_inside_git_repo(parPath) + reply = call_git("rev-parse --is-inside-work-tree 2> /dev/null || echo false", work_dir: parPath) + return "true" == reply +end + +class FlatGit + def initialize(parCloneDir) + unless File.exists?(parCloneDir) && File.directory?(parCloneDir) then + raise ArgumentError, "Specified path doesn't exist or is not a valid directory" + end + + if is_inside_git_repo(sanitized(parCloneDir)) then + raise ArgumentError, "Specified path is invalid because it appears to be inside a git repository" + end + + @clone_dir_abs = Pathname.new(parCloneDir).realpath + @clone_dir = parCloneDir + end + + def clone_submodules() + inplace_submodules = if File.file?("flat_git.yml") then + local_settings = YAML.load_file("flat_git.yml") + lst = local_settings["inplace_submodules"] + lst = [lst] if lst.is_a?(String) + lst.is_a?(Array) && lst || Array.new + else + Array.new + end + + submodules_info(current_remote_url()).each_value do |submod| + next unless submod.status == "-" + + guessed_clone_dir = File.join(@clone_dir, submod.name) + abs_guessed_clone_dir = File.join(@clone_dir_abs, submod.name) + if inplace_submodules.include?(submod.name) then + success = call_git(["submodule", "update", "--init", sanitized(submod.path)], false) + return false unless success + else + if !File.exists?(guessed_clone_dir) || Dir.entries(guessed_clone_dir).empty? then + success = call_git(["clone", sanitized(submod.url)], false, work_dir: sanitized(@clone_dir_abs)) + return false unless success + + if File.exists?(abs_guessed_clone_dir) && File.directory?(abs_guessed_clone_dir) then + call_git(["config", "core.worktree", ".."], work_dir: sanitized(abs_guessed_clone_dir)) + call_git(["reset", "--hard", sanitized(submod.hash)], work_dir: sanitized(abs_guessed_clone_dir)) + + Dir.chdir(abs_guessed_clone_dir) do |path| + if File.file?(".gitmodules") then + success = clone_submodules() + return false unless success + end + end + else + raise RuntimeError, "Unable to guess where git just cloned the repo" + end + end + call_git(["submodule", "init", sanitized(submod.path)]) + File.open(File.join(submod.path, ".git"), 'w') {|file| file.puts("gitdir: #{File.join(abs_guessed_clone_dir, ".git")}")} + end + end + return true + end +end + +unless ARGV.length == 1 then + $stderr.puts "Wrong number of arguments" + exit 2 +end + +exit(FlatGit.new(ARGV[0]).clone_submodules() ? 1 : 0)