King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2024-11-25 00:53:43 +00:00
Code Issues Releases Wiki Activity

New ScanIterator class.

Browse source 
Not yet tested and only supporting the SCAN command for now, more to
come.
Also includes some refactoring that was needed to make everything work.
This commit is contained in:
King_DuckZ 2016-06-10 20:33:11 +02:00
parent 04b667485e
commit c1e79c435b
10 changed files with 423 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
include/helpers/has_method.hpp Normal file
View file

@ -0,0 +1,54 @@
/*
* Copyright 2015 Michele "King_DuckZ" Santullo
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef idFBC29C5127784D35BE62F7BAC16E3687
#define idFBC29C5127784D35BE62F7BAC16E3687
#define define_has_method(method_name,pretty_name) \
template <typename T> \
struct Has ## pretty_name ## Method { \
private: \
struct TrueType { int a[2]; }; \
typedef int FalseType; \
template <typename C> static TrueType has_method ( decltype(&C::method_name) ); \
template <typename C> static FalseType has_method ( ... ); \
public: \
enum { value = sizeof(has_method<T>(0)) == sizeof(TrueType) }; \
}
#define define_has_typedef(typedef_name,pretty_name) \
template <typename T> \
struct Has ## pretty_name ## Typedef { \
private: \
struct TrueType { int a[2]; }; \
typedef int FalseType; \
template <typename C> static TrueType has_typedef ( const typename C::typedef_name* ); \
template <typename C> static FalseType has_typedef ( ... ); \
public: \
enum { value = sizeof(has_typedef<T>(nullptr)) == sizeof(TrueType) }; \
}
#define define_has_enum(enum_name,pretty_name) \
template <typename T> \
struct Has ## pretty_name ## Enum { \
private: \
struct TrueType { int a[2]; }; \
typedef int FalseType; \
template <typename C> static TrueType has_enum ( decltype(C::enum_name)* ); \
template <typename C> static FalseType has_enum ( ... ); \
public:\
enum { value = sizeof(has_enum<T>(nullptr)) == sizeof(TrueType) }; \
}
#endif

2
src/backends/redis/CMakeLists.txt
View file

@ -5,6 +5,8 @@ find_package(hiredis 0.11.0 REQUIRED)
add_library(${PROJECT_NAME} SHARED
backend_redis.cpp
command.cpp
scan_iterator.cpp
reply.cpp
)
target_include_directories(${PROJECT_NAME} SYSTEM

3
src/backends/redis/backend_redis.cpp
View file

@ -89,6 +89,9 @@ namespace dindb {
}
void BackendRedis::tag_files (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
for (const auto& file_path : m_redis.scan()) {
std::cout << file_path << '\n';
}
}
void BackendRedis::delete_tags (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {

17
src/backends/redis/command.cpp
View file

@ -23,7 +23,6 @@
#include <algorithm>
#include <stdexcept>
#include <boost/iterator/transform_iterator.hpp>
#include <boost/variant/get.hpp>
namespace redis {
namespace {
@ -49,18 +48,6 @@ namespace redis {
}
} //unnamed namespace
long long get_integer (const RedisReplyType& parReply) {
return boost::get<long long>(parReply);
}
std::string get_string (const RedisReplyType& parReply) {
return boost::get<std::string>(parReply);
}
std::vector<RedisReplyType> get_array (const RedisReplyType& parReply) {
return boost::get<std::vector<RedisReplyType>>(parReply);
}
Command::Command (std::string&& parAddress, uint16_t parPort) :
m_conn(nullptr, &redisFree),
m_address(std::move(parAddress)),
@ -138,4 +125,8 @@ namespace redis {
bool Command::is_connected() const {
return m_conn and not m_conn->err;
}
auto Command::scan() -> scan_range {
return scan_range(scan_iterator(this, false), scan_iterator(this, true));
}
} //namespace redis

44
src/backends/redis/command.hpp
View file

@ -19,47 +19,26 @@
#define idD83EEBFC927840C6B9F32D61A1D1E582
#include "arg_to_bin_safe.hpp"
#include "scan_iterator.hpp"
#include "reply.hpp"
#include <array>
#include <memory>
#include <string>
#include <cstdint>
#include <cstddef>
#include <cassert>
#include <boost/variant/variant.hpp>
#include <boost/variant/recursive_wrapper.hpp>
#include <vector>
#include <utility>
#include <boost/range/iterator_range_core.hpp>
struct redisContext;
namespace redis {
class RedisReplyType;
namespace implem {
using RedisVariantType = boost::variant<
long long,
std::string,
boost::recursive_wrapper<std::vector<RedisReplyType>>
>;
enum RedisVariantTypes {
RedisVariantType_Integer = 0,
RedisVariantType_String,
RedisVariantType_Array
};
} //namespace implem
struct RedisReplyType : implem::RedisVariantType {
using base_class = implem::RedisVariantType;
RedisReplyType ( void ) = default;
RedisReplyType ( long long parVal ) : base_class(parVal) {}
RedisReplyType ( std::string&& parVal ) : base_class(std::move(parVal)) {}
RedisReplyType ( std::vector<RedisReplyType>&& parVal ) : base_class(std::move(parVal)) {}
~RedisReplyType ( void ) noexcept = default;
};
class Command {
public:
typedef ScanIterator<std::string, ScanSingleValues<std::string>> scan_iterator;
typedef boost::iterator_range<scan_iterator> scan_range;
Command ( std::string&& parAddress, uint16_t parPort );
~Command ( void ) noexcept;
@ -71,6 +50,9 @@ namespace redis {
template <typename... Args>
RedisReplyType run ( const char* parCommand, Args&&... parArgs );
//Single Redis command wrappers
scan_range scan ( void );
private:
using RedisConnection = std::unique_ptr<redisContext, void(*)(redisContext*)>;
@ -87,10 +69,6 @@ namespace redis {
using CharPointerArray = std::array<const char*, arg_count>;
using LengthArray = std::array<std::size_t, arg_count>;
CharPointerArray arguments;
LengthArray lengths;
assert(false); //TODO write implementation
return this->run_pvt(
parCommand,
static_cast<int>(arg_count),
@ -98,10 +76,6 @@ namespace redis {
LengthArray{ implem::arg_to_bin_safe_length(std::forward<Args>(parArgs))... }.data()
);
}
long long get_integer ( const RedisReplyType& parReply );
std::string get_string ( const RedisReplyType& parReply );
std::vector<RedisReplyType> get_array ( const RedisReplyType& parReply );
} //namespace redis
#endif

52
src/backends/redis/reply.cpp Normal file
View file

@ -0,0 +1,52 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "reply.hpp"
#include <boost/variant/get.hpp>
namespace redis {
const long long& get_integer (const RedisReplyType& parReply) {
return boost::get<long long>(parReply);
}
const std::string& get_string (const RedisReplyType& parReply) {
return boost::get<std::string>(parReply);
}
const std::vector<RedisReplyType>& get_array (const RedisReplyType& parReply) {
return boost::get<std::vector<RedisReplyType>>(parReply);
}
template <>
const std::string& get<std::string> (const RedisReplyType& parReply) {
return get_string(parReply);
}
template <>
const std::vector<RedisReplyType>& get<std::vector<RedisReplyType>> (const RedisReplyType& parReply) {
return get_array(parReply);
}
template <>
const long long& get<long long> (const RedisReplyType& parReply) {
return get_integer(parReply);
}
template const std::string& get<std::string> ( const RedisReplyType& parReply );
template const std::vector<RedisReplyType>& get<std::vector<RedisReplyType>> ( const RedisReplyType& parReply );
template const long long& get<long long> ( const RedisReplyType& parReply );
} //namespace redis

60
src/backends/redis/reply.hpp Normal file
View file

@ -0,0 +1,60 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id93FA96E3071745D9A1E45D4D29B9F7D0
#define id93FA96E3071745D9A1E45D4D29B9F7D0
#include <boost/variant/variant.hpp>
#include <boost/variant/recursive_wrapper.hpp>
#include <string>
namespace redis {
class RedisReplyType;
namespace implem {
using RedisVariantType = boost::variant<
long long,
std::string,
boost::recursive_wrapper<std::vector<RedisReplyType>>
>;
enum RedisVariantTypes {
RedisVariantType_Integer = 0,
RedisVariantType_String,
RedisVariantType_Array,
RedisVariantType_Bool
};
} //namespace implem
struct RedisReplyType : implem::RedisVariantType {
using base_class = implem::RedisVariantType;
RedisReplyType ( void ) = default;
RedisReplyType ( long long parVal ) : base_class(parVal) {}
RedisReplyType ( std::string&& parVal ) : base_class(std::move(parVal)) {}
RedisReplyType ( std::vector<RedisReplyType>&& parVal ) : base_class(std::move(parVal)) {}
~RedisReplyType ( void ) noexcept = default;
};
const long long& get_integer ( const RedisReplyType& parReply );
const std::string& get_string ( const RedisReplyType& parReply );
const std::vector<RedisReplyType>& get_array ( const RedisReplyType& parReply );
template <typename T>
const T& get ( const RedisReplyType& parReply );
} //namespace redis
#endif

46
src/backends/redis/scan_iterator.cpp Normal file
View file

@ -0,0 +1,46 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "scan_iterator.hpp"
#include "command.hpp"
#include <cassert>
#include <ciso646>
#include <boost/lexical_cast.hpp>
#include <string>
namespace redis {
namespace implem {
ScanIteratorBaseClass::ScanIteratorBaseClass (Command* parCommand) :
m_command(parCommand)
{
assert(m_command);
assert(m_command->is_connected());
}
bool ScanIteratorBaseClass::is_connected() const {
return m_command and m_command->is_connected();
}
RedisReplyType ScanIteratorBaseClass::run (const char* parCommand, long long parScanContext) {
return m_command->run(parCommand, boost::lexical_cast<std::string>(parScanContext));
}
RedisReplyType ScanIteratorBaseClass::run (const char* parCommand, const std::string& parParameter, long long parScanContext) {
return m_command->run(parCommand, parParameter, boost::lexical_cast<std::string>(parScanContext));
}
} //namespace implem
} //namespace redis

95
src/backends/redis/scan_iterator.hpp Normal file
View file

@ -0,0 +1,95 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id774125B851514A26BD7C2AD1D804D732
#define id774125B851514A26BD7C2AD1D804D732
#include "reply.hpp"
#include "helpers/has_method.hpp"
#include <boost/iterator/iterator_facade.hpp>
#include <type_traits>
#include <vector>
#include <cstddef>
namespace redis {
template <typename V, typename ValueFetch>
class ScanIterator;
class Command;
namespace implem {
template <typename V, typename ValueFetch>
using ScanIteratorBaseIterator = boost::iterator_facade<ScanIterator<V, ValueFetch>, const V, boost::forward_traversal_tag>;
class ScanIteratorBaseClass {
protected:
explicit ScanIteratorBaseClass ( Command* parCommand );
~ScanIteratorBaseClass ( void ) noexcept = default;
bool is_connected ( void ) const;
RedisReplyType run ( const char* parCommand, long long parScanContext );
RedisReplyType run ( const char* parCommand, const std::string& parParameter, long long parScanContext );
bool is_equal ( const ScanIteratorBaseClass& parOther ) const { return m_command == parOther.m_command; }
private:
Command* m_command;
};
} //namespace implem
template <typename V, typename ValueFetch>
class ScanIterator : private implem::ScanIteratorBaseClass, public implem::ScanIteratorBaseIterator<V, ValueFetch> {
friend class boost::iterator_core_access;
typedef implem::ScanIteratorBaseIterator<V, ValueFetch> base_iterator;
public:
typedef typename base_iterator::difference_type difference_type;
typedef typename base_iterator::value_type value_type;
typedef typename base_iterator::pointer pointer;
typedef typename base_iterator::reference reference;
typedef typename base_iterator::iterator_category iterator_category;
ScanIterator ( Command* parCommand, bool parEnd );
private:
define_has_method(scan_target, ScanTarget);
template <typename T>
RedisReplyType forward_scan_command ( typename std::enable_if<HasScanTargetMethod<T>::value, int>::type parDummy );
template <typename T>
RedisReplyType forward_scan_command ( typename std::enable_if<not HasScanTargetMethod<T>::value, int>::type parDummy );
void increment ( void );
bool equal ( const ScanIterator& parOther ) const;
const V& dereference ( void ) const;
std::vector<RedisReplyType> m_reply;
long long m_scan_context;
std::size_t m_curr_index;
};
template <typename T>
struct ScanSingleValues {
static constexpr const char* command ( void ) { return "SCAN"; }
static constexpr const std::size_t step = 1;
static const T& make_value ( const RedisReplyType* parItem );
};
} //namespace redis
#include "scan_iterator.inl"
#endif

98
src/backends/redis/scan_iterator.inl Normal file
View file

@ -0,0 +1,98 @@
/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "command.hpp"
#include <cassert>
#include <ciso646>
namespace redis {
namespace implem {
} //namespace implem
template <typename V, typename ValueFetch>
ScanIterator<V, ValueFetch>::ScanIterator (Command* parCommand, bool parEnd) :
implem::ScanIteratorBaseClass(parCommand),
m_reply(),
m_scan_context(0),
m_curr_index(0)
{
if (not parEnd)
this->increment();
}
template <typename V, typename ValueFetch>
void ScanIterator<V, ValueFetch>::increment() {
static_assert(ValueFetch::step > 0, "Can't have an increase step of 0");
if (m_curr_index + ValueFetch::step < m_reply.size()) {
m_curr_index += ValueFetch::step;
}
else {
std::vector<RedisReplyType> array_reply;
long long new_context;
do {
auto whole_reply = this->forward_scan_command<ValueFetch>(0);
array_reply = get_array(whole_reply);
assert(2 == array_reply.size());
assert(array_reply.size() % ValueFetch::step == 0);
new_context = get_integer(array_reply[0]);
} while (new_context and array_reply.empty());
m_reply = get_array(array_reply[1]);
m_scan_context = new_context;
m_curr_index = 0;
}
}
template <typename V, typename ValueFetch>
bool ScanIterator<V, ValueFetch>::equal (const ScanIterator& parOther) const {
return
(&parOther == this) or
(
implem::ScanIteratorBaseClass::is_equal(parOther) and
(m_scan_context == parOther.m_scan_context)
);
}
template <typename V, typename ValueFetch>
const V& ScanIterator<V, ValueFetch>::dereference() const {
assert(m_scan_context);
assert(not m_reply.empty());
assert(m_curr_index < m_reply.size());
return ValueFetch::make_value(m_reply.data() + m_curr_index);
}
template <typename V, typename ValueFetch>
template <typename T>
RedisReplyType ScanIterator<V, ValueFetch>::forward_scan_command (typename std::enable_if<HasScanTargetMethod<T>::value, int>::type) {
return implem::ScanIteratorBaseClass::run(T::command(), T::scan_target(), m_scan_context);
}
template <typename V, typename ValueFetch>
template <typename T>
RedisReplyType ScanIterator<V, ValueFetch>::forward_scan_command (typename std::enable_if<not HasScanTargetMethod<T>::value, int>::type) {
return implem::ScanIteratorBaseClass::run(T::command(), m_scan_context);
}
template <typename T>
const T& ScanSingleValues<T>::make_value (const RedisReplyType* parItem) {
assert(parItem);
return get<T>(*parItem);
}
} //namespace redis