From c1e79c435b4ab74ea14c001f28fdcaba56f62275 Mon Sep 17 00:00:00 2001 From: King_DuckZ Date: Fri, 10 Jun 2016 20:33:11 +0200 Subject: [PATCH] New ScanIterator class. Not yet tested and only supporting the SCAN command for now, more to come. Also includes some refactoring that was needed to make everything work. --- include/helpers/has_method.hpp | 54 +++++++++++++++ src/backends/redis/CMakeLists.txt | 2 + src/backends/redis/backend_redis.cpp | 3 + src/backends/redis/command.cpp | 17 ++--- src/backends/redis/command.hpp | 44 +++---------- src/backends/redis/reply.cpp | 52 +++++++++++++++ src/backends/redis/reply.hpp | 60 +++++++++++++++++ src/backends/redis/scan_iterator.cpp | 46 +++++++++++++ src/backends/redis/scan_iterator.hpp | 95 +++++++++++++++++++++++++++ src/backends/redis/scan_iterator.inl | 98 ++++++++++++++++++++++++++++ 10 files changed, 423 insertions(+), 48 deletions(-) create mode 100644 include/helpers/has_method.hpp create mode 100644 src/backends/redis/reply.cpp create mode 100644 src/backends/redis/reply.hpp create mode 100644 src/backends/redis/scan_iterator.cpp create mode 100644 src/backends/redis/scan_iterator.hpp create mode 100644 src/backends/redis/scan_iterator.inl diff --git a/include/helpers/has_method.hpp b/include/helpers/has_method.hpp new file mode 100644 index 0000000..d641771 --- /dev/null +++ b/include/helpers/has_method.hpp @@ -0,0 +1,54 @@ +/* + * Copyright 2015 Michele "King_DuckZ" Santullo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef idFBC29C5127784D35BE62F7BAC16E3687 +#define idFBC29C5127784D35BE62F7BAC16E3687 + +#define define_has_method(method_name,pretty_name) \ + template \ + struct Has ## pretty_name ## Method { \ + private: \ + struct TrueType { int a[2]; }; \ + typedef int FalseType; \ + template static TrueType has_method ( decltype(&C::method_name) ); \ + template static FalseType has_method ( ... ); \ + public: \ + enum { value = sizeof(has_method(0)) == sizeof(TrueType) }; \ + } +#define define_has_typedef(typedef_name,pretty_name) \ + template \ + struct Has ## pretty_name ## Typedef { \ + private: \ + struct TrueType { int a[2]; }; \ + typedef int FalseType; \ + template static TrueType has_typedef ( const typename C::typedef_name* ); \ + template static FalseType has_typedef ( ... ); \ + public: \ + enum { value = sizeof(has_typedef(nullptr)) == sizeof(TrueType) }; \ + } +#define define_has_enum(enum_name,pretty_name) \ + template \ + struct Has ## pretty_name ## Enum { \ + private: \ + struct TrueType { int a[2]; }; \ + typedef int FalseType; \ + template static TrueType has_enum ( decltype(C::enum_name)* ); \ + template static FalseType has_enum ( ... ); \ + public:\ + enum { value = sizeof(has_enum(nullptr)) == sizeof(TrueType) }; \ + } + +#endif diff --git a/src/backends/redis/CMakeLists.txt b/src/backends/redis/CMakeLists.txt index 8a614df..6683d00 100644 --- a/src/backends/redis/CMakeLists.txt +++ b/src/backends/redis/CMakeLists.txt @@ -5,6 +5,8 @@ find_package(hiredis 0.11.0 REQUIRED) add_library(${PROJECT_NAME} SHARED backend_redis.cpp command.cpp + scan_iterator.cpp + reply.cpp ) target_include_directories(${PROJECT_NAME} SYSTEM diff --git a/src/backends/redis/backend_redis.cpp b/src/backends/redis/backend_redis.cpp index 1c28f15..00adefe 100644 --- a/src/backends/redis/backend_redis.cpp +++ b/src/backends/redis/backend_redis.cpp @@ -89,6 +89,9 @@ namespace dindb { } void BackendRedis::tag_files (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) { + for (const auto& file_path : m_redis.scan()) { + std::cout << file_path << '\n'; + } } void BackendRedis::delete_tags (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) { diff --git a/src/backends/redis/command.cpp b/src/backends/redis/command.cpp index f3efb90..f2de461 100644 --- a/src/backends/redis/command.cpp +++ b/src/backends/redis/command.cpp @@ -23,7 +23,6 @@ #include #include #include -#include namespace redis { namespace { @@ -49,18 +48,6 @@ namespace redis { } } //unnamed namespace - long long get_integer (const RedisReplyType& parReply) { - return boost::get(parReply); - } - - std::string get_string (const RedisReplyType& parReply) { - return boost::get(parReply); - } - - std::vector get_array (const RedisReplyType& parReply) { - return boost::get>(parReply); - } - Command::Command (std::string&& parAddress, uint16_t parPort) : m_conn(nullptr, &redisFree), m_address(std::move(parAddress)), @@ -138,4 +125,8 @@ namespace redis { bool Command::is_connected() const { return m_conn and not m_conn->err; } + + auto Command::scan() -> scan_range { + return scan_range(scan_iterator(this, false), scan_iterator(this, true)); + } } //namespace redis diff --git a/src/backends/redis/command.hpp b/src/backends/redis/command.hpp index 025780d..0dbb3aa 100644 --- a/src/backends/redis/command.hpp +++ b/src/backends/redis/command.hpp @@ -19,47 +19,26 @@ #define idD83EEBFC927840C6B9F32D61A1D1E582 #include "arg_to_bin_safe.hpp" +#include "scan_iterator.hpp" +#include "reply.hpp" #include #include #include #include #include #include -#include -#include #include #include +#include struct redisContext; namespace redis { - class RedisReplyType; - - namespace implem { - using RedisVariantType = boost::variant< - long long, - std::string, - boost::recursive_wrapper> - >; - enum RedisVariantTypes { - RedisVariantType_Integer = 0, - RedisVariantType_String, - RedisVariantType_Array - }; - } //namespace implem - - struct RedisReplyType : implem::RedisVariantType { - using base_class = implem::RedisVariantType; - - RedisReplyType ( void ) = default; - RedisReplyType ( long long parVal ) : base_class(parVal) {} - RedisReplyType ( std::string&& parVal ) : base_class(std::move(parVal)) {} - RedisReplyType ( std::vector&& parVal ) : base_class(std::move(parVal)) {} - ~RedisReplyType ( void ) noexcept = default; - }; - class Command { public: + typedef ScanIterator> scan_iterator; + typedef boost::iterator_range scan_range; + Command ( std::string&& parAddress, uint16_t parPort ); ~Command ( void ) noexcept; @@ -71,6 +50,9 @@ namespace redis { template RedisReplyType run ( const char* parCommand, Args&&... parArgs ); + //Single Redis command wrappers + scan_range scan ( void ); + private: using RedisConnection = std::unique_ptr; @@ -87,10 +69,6 @@ namespace redis { using CharPointerArray = std::array; using LengthArray = std::array; - CharPointerArray arguments; - LengthArray lengths; - assert(false); //TODO write implementation - return this->run_pvt( parCommand, static_cast(arg_count), @@ -98,10 +76,6 @@ namespace redis { LengthArray{ implem::arg_to_bin_safe_length(std::forward(parArgs))... }.data() ); } - - long long get_integer ( const RedisReplyType& parReply ); - std::string get_string ( const RedisReplyType& parReply ); - std::vector get_array ( const RedisReplyType& parReply ); } //namespace redis #endif diff --git a/src/backends/redis/reply.cpp b/src/backends/redis/reply.cpp new file mode 100644 index 0000000..6020d71 --- /dev/null +++ b/src/backends/redis/reply.cpp @@ -0,0 +1,52 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "reply.hpp" +#include + +namespace redis { + const long long& get_integer (const RedisReplyType& parReply) { + return boost::get(parReply); + } + + const std::string& get_string (const RedisReplyType& parReply) { + return boost::get(parReply); + } + + const std::vector& get_array (const RedisReplyType& parReply) { + return boost::get>(parReply); + } + + template <> + const std::string& get (const RedisReplyType& parReply) { + return get_string(parReply); + } + + template <> + const std::vector& get> (const RedisReplyType& parReply) { + return get_array(parReply); + } + + template <> + const long long& get (const RedisReplyType& parReply) { + return get_integer(parReply); + } + + template const std::string& get ( const RedisReplyType& parReply ); + template const std::vector& get> ( const RedisReplyType& parReply ); + template const long long& get ( const RedisReplyType& parReply ); +} //namespace redis diff --git a/src/backends/redis/reply.hpp b/src/backends/redis/reply.hpp new file mode 100644 index 0000000..eb9469f --- /dev/null +++ b/src/backends/redis/reply.hpp @@ -0,0 +1,60 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id93FA96E3071745D9A1E45D4D29B9F7D0 +#define id93FA96E3071745D9A1E45D4D29B9F7D0 + +#include +#include +#include + +namespace redis { + class RedisReplyType; + + namespace implem { + using RedisVariantType = boost::variant< + long long, + std::string, + boost::recursive_wrapper> + >; + enum RedisVariantTypes { + RedisVariantType_Integer = 0, + RedisVariantType_String, + RedisVariantType_Array, + RedisVariantType_Bool + }; + } //namespace implem + + struct RedisReplyType : implem::RedisVariantType { + using base_class = implem::RedisVariantType; + + RedisReplyType ( void ) = default; + RedisReplyType ( long long parVal ) : base_class(parVal) {} + RedisReplyType ( std::string&& parVal ) : base_class(std::move(parVal)) {} + RedisReplyType ( std::vector&& parVal ) : base_class(std::move(parVal)) {} + ~RedisReplyType ( void ) noexcept = default; + }; + + const long long& get_integer ( const RedisReplyType& parReply ); + const std::string& get_string ( const RedisReplyType& parReply ); + const std::vector& get_array ( const RedisReplyType& parReply ); + + template + const T& get ( const RedisReplyType& parReply ); +} //namespace redis + +#endif diff --git a/src/backends/redis/scan_iterator.cpp b/src/backends/redis/scan_iterator.cpp new file mode 100644 index 0000000..00736a5 --- /dev/null +++ b/src/backends/redis/scan_iterator.cpp @@ -0,0 +1,46 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "scan_iterator.hpp" +#include "command.hpp" +#include +#include +#include +#include + +namespace redis { + namespace implem { + ScanIteratorBaseClass::ScanIteratorBaseClass (Command* parCommand) : + m_command(parCommand) + { + assert(m_command); + assert(m_command->is_connected()); + } + + bool ScanIteratorBaseClass::is_connected() const { + return m_command and m_command->is_connected(); + } + + RedisReplyType ScanIteratorBaseClass::run (const char* parCommand, long long parScanContext) { + return m_command->run(parCommand, boost::lexical_cast(parScanContext)); + } + + RedisReplyType ScanIteratorBaseClass::run (const char* parCommand, const std::string& parParameter, long long parScanContext) { + return m_command->run(parCommand, parParameter, boost::lexical_cast(parScanContext)); + } + } //namespace implem +} //namespace redis diff --git a/src/backends/redis/scan_iterator.hpp b/src/backends/redis/scan_iterator.hpp new file mode 100644 index 0000000..98bcb52 --- /dev/null +++ b/src/backends/redis/scan_iterator.hpp @@ -0,0 +1,95 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id774125B851514A26BD7C2AD1D804D732 +#define id774125B851514A26BD7C2AD1D804D732 + +#include "reply.hpp" +#include "helpers/has_method.hpp" +#include +#include +#include +#include + +namespace redis { + template + class ScanIterator; + + class Command; + + namespace implem { + template + using ScanIteratorBaseIterator = boost::iterator_facade, const V, boost::forward_traversal_tag>; + + class ScanIteratorBaseClass { + protected: + explicit ScanIteratorBaseClass ( Command* parCommand ); + ~ScanIteratorBaseClass ( void ) noexcept = default; + + bool is_connected ( void ) const; + RedisReplyType run ( const char* parCommand, long long parScanContext ); + RedisReplyType run ( const char* parCommand, const std::string& parParameter, long long parScanContext ); + + bool is_equal ( const ScanIteratorBaseClass& parOther ) const { return m_command == parOther.m_command; } + + private: + Command* m_command; + }; + } //namespace implem + + template + class ScanIterator : private implem::ScanIteratorBaseClass, public implem::ScanIteratorBaseIterator { + friend class boost::iterator_core_access; + typedef implem::ScanIteratorBaseIterator base_iterator; + public: + typedef typename base_iterator::difference_type difference_type; + typedef typename base_iterator::value_type value_type; + typedef typename base_iterator::pointer pointer; + typedef typename base_iterator::reference reference; + typedef typename base_iterator::iterator_category iterator_category; + + ScanIterator ( Command* parCommand, bool parEnd ); + + private: + define_has_method(scan_target, ScanTarget); + + template + RedisReplyType forward_scan_command ( typename std::enable_if::value, int>::type parDummy ); + template + RedisReplyType forward_scan_command ( typename std::enable_if::value, int>::type parDummy ); + + void increment ( void ); + bool equal ( const ScanIterator& parOther ) const; + const V& dereference ( void ) const; + + std::vector m_reply; + long long m_scan_context; + std::size_t m_curr_index; + }; + + template + struct ScanSingleValues { + static constexpr const char* command ( void ) { return "SCAN"; } + static constexpr const std::size_t step = 1; + + static const T& make_value ( const RedisReplyType* parItem ); + }; +} //namespace redis + +#include "scan_iterator.inl" + +#endif diff --git a/src/backends/redis/scan_iterator.inl b/src/backends/redis/scan_iterator.inl new file mode 100644 index 0000000..ea7f7f3 --- /dev/null +++ b/src/backends/redis/scan_iterator.inl @@ -0,0 +1,98 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "command.hpp" +#include +#include + +namespace redis { + namespace implem { + } //namespace implem + + template + ScanIterator::ScanIterator (Command* parCommand, bool parEnd) : + implem::ScanIteratorBaseClass(parCommand), + m_reply(), + m_scan_context(0), + m_curr_index(0) + { + if (not parEnd) + this->increment(); + } + + template + void ScanIterator::increment() { + static_assert(ValueFetch::step > 0, "Can't have an increase step of 0"); + if (m_curr_index + ValueFetch::step < m_reply.size()) { + m_curr_index += ValueFetch::step; + } + else { + std::vector array_reply; + long long new_context; + + do { + auto whole_reply = this->forward_scan_command(0); + + array_reply = get_array(whole_reply); + assert(2 == array_reply.size()); + assert(array_reply.size() % ValueFetch::step == 0); + new_context = get_integer(array_reply[0]); + } while (new_context and array_reply.empty()); + + m_reply = get_array(array_reply[1]); + m_scan_context = new_context; + m_curr_index = 0; + } + } + + template + bool ScanIterator::equal (const ScanIterator& parOther) const { + return + (&parOther == this) or + ( + implem::ScanIteratorBaseClass::is_equal(parOther) and + (m_scan_context == parOther.m_scan_context) + ); + } + + template + const V& ScanIterator::dereference() const { + assert(m_scan_context); + assert(not m_reply.empty()); + assert(m_curr_index < m_reply.size()); + + return ValueFetch::make_value(m_reply.data() + m_curr_index); + } + + template + template + RedisReplyType ScanIterator::forward_scan_command (typename std::enable_if::value, int>::type) { + return implem::ScanIteratorBaseClass::run(T::command(), T::scan_target(), m_scan_context); + } + + template + template + RedisReplyType ScanIterator::forward_scan_command (typename std::enable_if::value, int>::type) { + return implem::ScanIteratorBaseClass::run(T::command(), m_scan_context); + } + + template + const T& ScanSingleValues::make_value (const RedisReplyType* parItem) { + assert(parItem); + return get(*parItem); + } +} //namespace redis