King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-07-03 14:14:11 +00:00
Code Issues Releases Wiki Activity

Move *scan function from Command to IncRedis.

Browse source 
Pass down IncRedis as required to fix the build.
This commit is contained in:
King_DuckZ 2016-07-12 10:55:45 +01:00
parent d0242e2721
commit 64b87c52bb
11 changed files with 82 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

24
src/backends/redis/backend_redis.cpp
View file

@ -129,31 +129,31 @@ namespace dindb {
}
void BackendRedis::tag_files (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
dindb::tag_files(m_redis.command(), m_tag_if_in_set, parFiles, parTags, parSet);
dindb::tag_files(m_redis, m_tag_if_in_set, parFiles, parTags, parSet);
}
void BackendRedis::tag_files (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
dindb::tag_files(m_redis.command(), m_tag_if_in_set, parRegexes, parTags, parSet);
dindb::tag_files(m_redis, m_tag_if_in_set, parRegexes, parTags, parSet);
}
void BackendRedis::delete_tags (const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
dindb::delete_tags(m_redis.command(), m_dele_tag_if_in_set, parFiles, parTags, parSet);
dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parFiles, parTags, parSet);
}
void BackendRedis::delete_tags (const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
dindb::delete_tags(m_redis.command(), m_dele_tag_if_in_set, parRegexes, parTags, parSet);
dindb::delete_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parTags, parSet);
}
void BackendRedis::delete_all_tags (const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
dindb::delete_all_tags(m_redis.command(), m_dele_tag_if_in_set, parFiles, parSet);
dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parFiles, parSet);
}
void BackendRedis::delete_all_tags (const std::vector<std::string>& parRegexes, GroupIDType parSet) {
dindb::delete_all_tags(m_redis.command(), m_dele_tag_if_in_set, parRegexes, parSet);
dindb::delete_all_tags(m_redis, m_dele_tag_if_in_set, parRegexes, parSet);
}
void BackendRedis::delete_group (const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
delete_group_from_db(m_redis.command(), m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf);
delete_group_from_db(m_redis, m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf);
}
void BackendRedis::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
@ -223,23 +223,23 @@ namespace dindb {
}
else {
const auto result_id = std::move(*hash_reply);
auto set_key_and_file_item = redis::range_as<FileRecordDataWithGroup>(m_redis.command().hscan(result_id));
auto set_key_and_file_item = redis::range_as<FileRecordDataWithGroup>(m_redis.hscan(result_id));
parItem = std::move(set_key_and_file_item.second);
const std::string group_key = std::move(set_key_and_file_item.first);
auto scan_range = m_redis.command().hscan(group_key);
auto scan_range = m_redis.hscan(group_key);
if (empty(scan_range)) {
return false;
}
else {
parSet = redis::range_as<mchlib::SetRecordDataFull>(m_redis.command().hscan(group_key));
parSet = redis::range_as<mchlib::SetRecordDataFull>(m_redis.hscan(group_key));
return true;
}
}
}
std::vector<LocatedItem> BackendRedis::locate_in_db (const std::string& parSearch, const TagList& parTags) {
return dindb::locate_in_db(m_redis.command(), parSearch, parTags);
return dindb::locate_in_db(m_redis, parSearch, parTags);
}
std::vector<LocatedItem> BackendRedis::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) {
@ -255,7 +255,7 @@ namespace dindb {
}
std::vector<GroupIDType> BackendRedis::find_all_sets() {
return dindb::find_all_sets(m_redis.command());
return dindb::find_all_sets(m_redis);
}
std::vector<dinhelp::MaxSizedArray<std::string, 4>> BackendRedis::find_set_details (const std::vector<GroupIDType>& parSets) {

16
src/backends/redis/command.cpp
View file

@ -77,22 +77,6 @@ namespace redis {
return m_local_data->async_connection.connection_error();
}
auto Command::scan (boost::string_ref parPattern) -> scan_range {
return scan_range(scan_iterator(this, false, parPattern), scan_iterator(this, true));
}
auto Command::hscan (boost::string_ref parKey, boost::string_ref parPattern) -> hscan_range {
return hscan_range(hscan_iterator(this, parKey, false, parPattern), hscan_iterator(this, parKey, true));
}
auto Command::sscan (boost::string_ref parKey, boost::string_ref parPattern) -> sscan_range {
return sscan_range(sscan_iterator(this, parKey, false, parPattern), sscan_iterator(this, parKey, true));
}
auto Command::zscan (boost::string_ref parKey, boost::string_ref parPattern) -> zscan_range {
return zscan_range(zscan_iterator(this, parKey, false, parPattern), zscan_iterator(this, parKey, true));
}
Batch Command::make_batch() {
assert(is_connected());
return Batch(&m_local_data->async_connection);

18
src/backends/redis/command.hpp
View file

@ -18,7 +18,6 @@
#ifndef idD83EEBFC927840C6B9F32D61A1D1E582
#define idD83EEBFC927840C6B9F32D61A1D1E582
#include "scan_iterator.hpp"
#include "reply.hpp"
#include "batch.hpp"
#include "redisConfig.h"
@ -30,8 +29,6 @@
#include <cassert>
#include <vector>
#include <utility>
#include <boost/range/iterator_range_core.hpp>
#include <boost/range/empty.hpp>
#include <boost/utility/string_ref.hpp>
#include <ciso646>
#include <stdexcept>
@ -39,15 +36,6 @@
namespace redis {
class Command {
public:
typedef ScanIterator<ScanSingleValues<std::string>> scan_iterator;
typedef boost::iterator_range<scan_iterator> scan_range;
typedef ScanIterator<ScanPairs<std::pair<std::string, std::string>, ScanCommands::HSCAN>> hscan_iterator;
typedef boost::iterator_range<hscan_iterator> hscan_range;
typedef ScanIterator<ScanSingleValuesInKey<std::string>> sscan_iterator;
typedef boost::iterator_range<sscan_iterator> sscan_range;
typedef ScanIterator<ScanPairs<std::pair<std::string, std::string>, ScanCommands::ZSCAN>> zscan_iterator;
typedef boost::iterator_range<zscan_iterator> zscan_range;
Command ( std::string&& parAddress, uint16_t parPort );
explicit Command ( std::string&& parSocket );
~Command ( void ) noexcept;
@ -66,12 +54,6 @@ namespace redis {
template <typename... Args>
Reply run ( const char* parCommand, Args&&... parArgs );
//Single Redis command wrappers
scan_range scan ( boost::string_ref parPattern=boost::string_ref() );
hscan_range hscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
sscan_range sscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
zscan_range zscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
private:
struct LocalData;

10
src/backends/redis/delete.cpp
View file

@ -17,7 +17,7 @@
#include "delete.hpp"
#include "tag.hpp"
#include "command.hpp"
#include "incredis.hpp"
#include "helpers/lexical_cast.hpp"
#include "helpers/sequence_bt.hpp"
#include "dindexerConfig.h"
@ -76,11 +76,11 @@ namespace dindb {
};
} //unnamed namespace
void delete_group_from_db (redis::Command& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
void delete_group_from_db (redis::IncRedis& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
using dinhelp::lexical_cast;
using IDRange = std::tuple<GroupIDType, FileIDType, FileIDType>;
auto set_batch = parRedis.make_batch();
auto set_batch = parRedis.command().make_batch();
auto dele_pair = confirm_dele(set_batch, parIDs, parConf);
assert(set_batch.replies_requested());
@ -121,13 +121,13 @@ namespace dindb {
delete_all_tags(parRedis, parDeleTagIfInSet, ids, set_id);
}
auto dele_batch = parRedis.make_batch();
auto dele_batch = parRedis.command().make_batch();
for (const auto& dele_tuple : ranges) {
const auto set_id = std::get<0>(dele_tuple);
const auto file_base_index = std::get<1>(dele_tuple);
const auto file_count = std::get<2>(dele_tuple);
auto hash_query_batch = parRedis.make_batch();
auto hash_query_batch = parRedis.command().make_batch();
for (FileIDType i = file_base_index; i < file_base_index + file_count; ++i) {
const auto file_key = PROGRAM_NAME ":file:" + lexical_cast<std::string>(i);
hash_query_batch.run("HGET", file_key, "hash");

4
src/backends/redis/delete.hpp
View file

@ -25,7 +25,7 @@
#include <map>
namespace redis {
class Command;
class IncRedis;
class Script;
} //namespace redis
@ -34,7 +34,7 @@ namespace dindb {
using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
void delete_group_from_db (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parDeleTagIfInSet,
redis::Script& parDeleHash,
const std::vector<GroupIDType>& parIDs,

8
src/backends/redis/find.cpp
View file

@ -16,7 +16,7 @@
*/
#include "find.hpp"
#include "command.hpp"
#include "incredis.hpp"
#include "helpers/lexical_cast.hpp"
#include "dindexerConfig.h"
#include "dindexer-core/split_tags.hpp"
@ -64,7 +64,7 @@ namespace dindb {
}
} //unnamed namespace
std::vector<GroupIDType> find_all_sets (redis::Command& parRedis) {
std::vector<GroupIDType> find_all_sets (redis::IncRedis& parRedis) {
using dincore::split_and_trim;
using dinhelp::lexical_cast;
@ -75,7 +75,7 @@ namespace dindb {
return retval;
}
std::vector<LocatedItem> locate_in_db (redis::Command& parRedis, const std::string& parSearch, const TagList& parTags) {
std::vector<LocatedItem> locate_in_db (redis::IncRedis& parRedis, const std::string& parSearch, const TagList& parTags) {
using dincore::split_and_trim;
using dinhelp::lexical_cast;
@ -87,7 +87,7 @@ namespace dindb {
ids.reserve(prefetch_count);
int curr_count = 0;
auto batch = parRedis.make_batch();
auto batch = parRedis.command().make_batch();
for (const auto& itm : parRedis.scan(PROGRAM_NAME ":file:*")) {
++curr_count;
batch.run("HMGET", itm, "path", "group_id", "tags");

6
src/backends/redis/find.hpp
View file

@ -22,12 +22,12 @@
#include <vector>
namespace redis {
class Command;
class IncRedis;
} //namespace redis
namespace dindb {
std::vector<GroupIDType> find_all_sets ( redis::Command& parRedis );
std::vector<LocatedItem> locate_in_db ( redis::Command& parRedis, const std::string& parSearch, const TagList& parTags );
std::vector<GroupIDType> find_all_sets ( redis::IncRedis& parRedis );
std::vector<LocatedItem> locate_in_db ( redis::IncRedis& parRedis, const std::string& parSearch, const TagList& parTags );
} //namespace dindb
#endif

16
src/backends/redis/incredis.cpp
View file

@ -77,6 +77,22 @@ namespace redis {
m_command.wait_for_disconnect();
}
auto IncRedis::scan (boost::string_ref parPattern) -> scan_range {
return scan_range(scan_iterator(&m_command, false, parPattern), scan_iterator(&m_command, true));
}
auto IncRedis::hscan (boost::string_ref parKey, boost::string_ref parPattern) -> hscan_range {
return hscan_range(hscan_iterator(&m_command, parKey, false, parPattern), hscan_iterator(&m_command, parKey, true));
}
auto IncRedis::sscan (boost::string_ref parKey, boost::string_ref parPattern) -> sscan_range {
return sscan_range(sscan_iterator(&m_command, parKey, false, parPattern), sscan_iterator(&m_command, parKey, true));
}
auto IncRedis::zscan (boost::string_ref parKey, boost::string_ref parPattern) -> zscan_range {
return zscan_range(zscan_iterator(&m_command, parKey, false, parPattern), zscan_iterator(&m_command, parKey, true));
}
auto IncRedis::hget (boost::string_ref parKey, boost::string_ref parField) -> opt_string {
return optional_string(m_command.run("HGET", parKey, parField));
}

18
src/backends/redis/incredis.hpp
View file

@ -19,14 +19,26 @@
#define id7D338900114548A890B1EECE0C4D3C4C
#include "command.hpp"
#include "scan_iterator.hpp"
#include <boost/optional.hpp>
#include <string>
#include <boost/utility/string_ref.hpp>
#include <vector>
#include <boost/range/iterator_range_core.hpp>
#include <boost/range/empty.hpp>
namespace redis {
class IncRedis {
public:
typedef ScanIterator<ScanSingleValues<std::string>> scan_iterator;
typedef boost::iterator_range<scan_iterator> scan_range;
typedef ScanIterator<ScanPairs<std::pair<std::string, std::string>, ScanCommands::HSCAN>> hscan_iterator;
typedef boost::iterator_range<hscan_iterator> hscan_range;
typedef ScanIterator<ScanSingleValuesInKey<std::string>> sscan_iterator;
typedef boost::iterator_range<sscan_iterator> sscan_range;
typedef ScanIterator<ScanPairs<std::pair<std::string, std::string>, ScanCommands::ZSCAN>> zscan_iterator;
typedef boost::iterator_range<zscan_iterator> zscan_range;
typedef boost::optional<std::string> opt_string;
typedef boost::optional<std::vector<opt_string>> opt_string_list;
@ -43,6 +55,12 @@ namespace redis {
Command& command ( void ) { return m_command; }
const Command& command ( void ) const { return m_command; }
//Scan
scan_range scan ( boost::string_ref parPattern=boost::string_ref() );
hscan_range hscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
sscan_range sscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
zscan_range zscan ( boost::string_ref parKey, boost::string_ref parPattern=boost::string_ref() );
//Hash
opt_string hget ( boost::string_ref parKey, boost::string_ref parField );
int hincrby ( boost::string_ref parKey, boost::string_ref parField, int parInc );

30
src/backends/redis/tag.cpp
View file

@ -16,7 +16,7 @@
*/
#include "tag.hpp"
#include "command.hpp"
#include "incredis.hpp"
#include "dindexerConfig.h"
#include "helpers/lexical_cast.hpp"
#include "dindexer-core/split_tags.hpp"
@ -54,10 +54,10 @@ namespace dindb {
return retval;
}
void run_id_based_script (redis::Command& parRedis, redis::Script& parScript, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void run_id_based_script (redis::IncRedis& parRedis, redis::Script& parScript, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
using dinhelp::lexical_cast;
auto batch = parRedis.make_batch();
auto batch = parRedis.command().make_batch();
const std::string set_id = lexical_cast<std::string>(parSet);
for (const auto file_id : parFiles) {
for (const auto &tag : parTags) {
@ -72,18 +72,18 @@ namespace dindb {
batch.throw_if_failed();
}
void run_regex_based_script(redis::Command& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void run_regex_based_script(redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
using dinhelp::lexical_cast;
const std::string set_id = lexical_cast<std::string>(parSet);
const auto regexes = compile_regexes(parRegexes);
for (const auto &itm : parRedis.scan(PROGRAM_NAME ":file:*")) {
const auto &file_key = itm;
const auto path = redis::get_string(parRedis.run("HGET", file_key, "path"));
const auto path = parRedis.hget(file_key, "path");
auto batch = parRedis.make_batch();
auto batch = parRedis.command().make_batch();
for (const auto &regex : regexes) {
if (not boost::regex_search(path, regex))
if (not path or not boost::regex_search(*path, regex))
continue;
for (const auto &tag : parTags) {
@ -105,24 +105,24 @@ namespace dindb {
}
} //unnamed namespace
void tag_files (redis::Command& parRedis, redis::Script& parTagIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
run_id_based_script(parRedis, parTagIfInSet, parFiles, parTags, parSet);
}
void tag_files (redis::Command& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void tag_files (redis::IncRedis& parRedis, redis::Script& parTagIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
run_regex_based_script(parRedis, parTagIfInSet, parRegexes, parTags, parSet);
}
void delete_tags (redis::Command& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
run_id_based_script(parRedis, parDeleIfInSet, parFiles, parTags, parSet);
}
void delete_tags (redis::Command& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
void delete_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, GroupIDType parSet) {
run_regex_based_script(parRedis, parDeleIfInSet, parRegexes, parTags, parSet);
}
void delete_all_tags (redis::Command& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
auto batch = parRedis.make_batch();
void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<FileIDType>& parFiles, GroupIDType parSet) {
auto batch = parRedis.command().make_batch();
for (const auto file_id : parFiles) {
const auto file_key = make_file_key(file_id);
batch.run("HGET", file_key, "tags");
@ -141,7 +141,7 @@ namespace dindb {
delete_tags(parRedis, parDeleIfInSet, parFiles, vec_dele_tags, parSet);
}
void delete_all_tags (redis::Command& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, GroupIDType parSet) {
void delete_all_tags (redis::IncRedis& parRedis, redis::Script& parDeleIfInSet, const std::vector<std::string>& parRegexes, GroupIDType parSet) {
using dinhelp::lexical_cast;
const auto regexes = compile_regexes(parRegexes);
@ -151,7 +151,7 @@ namespace dindb {
for (const auto& itm : parRedis.scan(PROGRAM_NAME ":file:*")) {
const auto& file_key = itm;
auto file_reply = parRedis.run("HMGET", file_key, "path", "tags", "group_id");
auto file_reply = parRedis.command().run("HMGET", file_key, "path", "tags", "group_id");
auto& file_replies = redis::get_array(file_reply);
assert(file_replies.size() == 3);
const auto group_id = lexical_cast<GroupIDType>(redis::get_string(file_replies[2]));

14
src/backends/redis/tag.hpp
View file

@ -23,20 +23,20 @@
#include <boost/utility/string_ref.hpp>
namespace redis {
class Command;
class IncRedis;
class Script;
} //namespace redis
namespace dindb {
void tag_files (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parTagIfInSet,
const std::vector<FileIDType>& parFiles,
const std::vector<boost::string_ref>& parTags,
GroupIDType parSet
);
void tag_files (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parTagIfInSet,
const std::vector<std::string>& parRegexes,
const std::vector<boost::string_ref>& parTags,
@ -44,27 +44,27 @@ namespace dindb {
);
void delete_tags (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parDeleIfInSet,
const std::vector<FileIDType>& parFiles,
const std::vector<boost::string_ref>& parTags,
GroupIDType parSet
);
void delete_tags (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parDeleIfInSet,
const std::vector<std::string>& parRegexes,
const std::vector<boost::string_ref>& parTags,
GroupIDType parSet
);
void delete_all_tags (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parDeleIfInSet,
const std::vector<FileIDType>& parFiles,
GroupIDType parSet
);
void delete_all_tags (
redis::Command& parRedis,
redis::IncRedis& parRedis,
redis::Script& parDeleIfInSet,
const std::vector<std::string>& parRegexes,
GroupIDType parSet