diff --git a/src/backends/redis/CMakeLists.txt b/src/backends/redis/CMakeLists.txt index 941b141..8f3db1e 100644 --- a/src/backends/redis/CMakeLists.txt +++ b/src/backends/redis/CMakeLists.txt @@ -17,6 +17,7 @@ add_library(${PROJECT_NAME} SHARED script_manager.cpp async_connection.cpp tag.cpp + delete.cpp ) target_include_directories(${PROJECT_NAME} SYSTEM @@ -57,6 +58,7 @@ configure_file( set(LUA_SCRIPTS tag_if_in_set.lua dele_tag_if_in_set.lua + dele_hash.lua ) install(TARGETS ${PROJECT_NAME} diff --git a/src/backends/redis/backend_redis.cpp b/src/backends/redis/backend_redis.cpp index b1ada5e..dcdd07b 100644 --- a/src/backends/redis/backend_redis.cpp +++ b/src/backends/redis/backend_redis.cpp @@ -23,6 +23,7 @@ #include "dindexerConfig.h" #include "helpers/stringize.h" #include "tag.hpp" +#include "delete.hpp" #include "record_data_adapt.hpp" #include #include @@ -108,6 +109,7 @@ namespace dindb { auto batch = m_redis.make_batch(); batch.run("SELECT", lexical_cast(m_database)); batch.run("CLIENT", "SETNAME", PROGRAM_NAME "_v" STRINGIZE(VERSION_MAJOR) "." STRINGIZE(VERSION_MINOR) "." STRINGIZE(VERSION_PATCH)); + batch.run("SCRIPT", "FLUSH"); batch.throw_if_failed(); } else { @@ -118,6 +120,7 @@ namespace dindb { m_tag_if_in_set = m_redis.make_script(read_script(m_lua_script_paths, "tag_if_in_set.lua")); m_dele_tag_if_in_set = m_redis.make_script(read_script(m_lua_script_paths, "dele_tag_if_in_set.lua")); + m_dele_hash = m_redis.make_script(read_script(m_lua_script_paths, "dele_hash.lua")); } void BackendRedis::disconnect() { @@ -149,6 +152,7 @@ namespace dindb { } void BackendRedis::delete_group (const std::vector& parIDs, ConfirmDeleCallback parConf) { + delete_group_from_db(m_redis, m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf); } void BackendRedis::write_files (const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) { @@ -173,7 +177,9 @@ namespace dindb { "disk_label", parSetData.disk_label, "fs_uuid", parSetData.fs_uuid, "type", parSetData.type, - "content_type", parSetData.content_type + "content_type", parSetData.content_type, + "base_file_id", lexical_cast(base_file_id), + "file_count", lexical_cast(parData.size()) ); for (auto z = base_file_id; z < casted_data_size + 1; ++z) { diff --git a/src/backends/redis/backend_redis.hpp b/src/backends/redis/backend_redis.hpp index bd24782..a3ed3dc 100644 --- a/src/backends/redis/backend_redis.hpp +++ b/src/backends/redis/backend_redis.hpp @@ -61,6 +61,7 @@ namespace dindb { redis::Command m_redis; redis::Script m_tag_if_in_set; redis::Script m_dele_tag_if_in_set; + redis::Script m_dele_hash; dincore::SearchPaths m_lua_script_paths; uint16_t m_database; }; diff --git a/src/backends/redis/dele_hash.lua b/src/backends/redis/dele_hash.lua new file mode 100644 index 0000000..078482e --- /dev/null +++ b/src/backends/redis/dele_hash.lua @@ -0,0 +1,27 @@ +-- Copyright 2015, 2016, Michele Santullo +-- This file is part of "dindexer". +-- +-- "dindexer" is free software: you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. +-- +-- "dindexer" is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with "dindexer". If not, see . + +local hash_key = KEYS[1] +local base_index = ARGV[1] +local file_count = ARGV[2] + +for z = base_index, base_index + file_count - 1 do + redis.call("SREM", hash_key, z) +end +if redis.call("SCARD", hash_key) == 0 then + redis.call("DEL", hash_key) +end +return nil diff --git a/src/backends/redis/delete.cpp b/src/backends/redis/delete.cpp new file mode 100644 index 0000000..1c0aa5e --- /dev/null +++ b/src/backends/redis/delete.cpp @@ -0,0 +1,152 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#include "delete.hpp" +#include "tag.hpp" +#include "command.hpp" +#include "helpers/lexical_cast.hpp" +#include "helpers/sequence_bt.hpp" +#include "dindexerConfig.h" +#include +#include +#include +#include +#include + +namespace dindb { + namespace { + std::pair confirm_dele (redis::Batch& parBatch, const std::vector& parIDs, ConfirmDeleCallback parConf) { + using dinhelp::lexical_cast; + + if (parIDs.empty()) + return std::make_pair(false, parIDs.size()); + + for (auto id : parIDs) { + const auto set_key = PROGRAM_NAME ":set:" + lexical_cast(id); + parBatch.run("HMGET", set_key, "base_file_id", "file_count", "name"); + } + + std::map set_dele_list; + std::size_t id_index = 0; + for (const auto& reply : parBatch.replies()) { + const auto res = redis::get_array(reply); + assert(res.size() == 3); + if (redis::RedisVariantType_Nil != res[0].which()) { + assert(redis::RedisVariantType_Nil != res[1].which()); + set_dele_list[parIDs[id_index]] = redis::get_string(res[2]); + } + ++id_index; + } + + if (set_dele_list.empty()) + return std::make_pair(false, set_dele_list.size()); + else + return std::make_pair(parConf(set_dele_list), set_dele_list.size()); + //return std::make_pair(true, set_dele_list.size()); + } + + template + void chunked_run_pvt (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey, dinhelp::bt::number_seq) { + for (IT i = 0; i < parCount / CHUNK; ++i) { + parBatch.run(parCommand, parMakeKey(i * CHUNK + parFrom + SVALS)...); + } + for (auto i = ((parFrom + parCount) / CHUNK) * CHUNK; i < parFrom + parCount; ++i) { + const auto key = parMakeKey(i); + parBatch.run(parCommand, key); + } + } + + template + void chunked_run (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey) { + chunked_run_pvt(parBatch, parCommand, parFrom, parCount, parMakeKey, dinhelp::bt::number_range()); + }; + } //unnamed namespace + + void delete_group_from_db (redis::Command& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector& parIDs, ConfirmDeleCallback parConf) { + using dinhelp::lexical_cast; + using IDRange = std::tuple; + + auto set_batch = parRedis.make_batch(); + + auto dele_pair = confirm_dele(set_batch, parIDs, parConf); + assert(set_batch.replies_requested()); + if (not dele_pair.first) + return; + + std::vector ranges; + ranges.reserve(dele_pair.second); + std::size_t id_index = 0; + for (const auto& reply : set_batch.replies()) { + const auto res = redis::get_array(reply); + if (redis::RedisVariantType_Nil != res[0].which()) { + ranges.push_back( + IDRange( + parIDs[id_index], + lexical_cast(redis::get_string(res[0])), + lexical_cast(redis::get_string(res[1])) + ) + ); + } + ++id_index; + } + assert(ranges.size() == dele_pair.second); + + for (const auto& dele_tuple : ranges) { + const auto set_id = std::get<0>(dele_tuple); + const auto file_base_index = std::get<1>(dele_tuple); + const auto file_count = std::get<2>(dele_tuple); + + std::vector ids; + ids.reserve(file_count); + std::copy( + boost::counting_iterator(file_base_index), + boost::counting_iterator(file_base_index + file_count), + std::back_inserter(ids) + ); + + delete_all_tags(parRedis, parDeleTagIfInSet, ids, set_id); + } + + auto dele_batch = parRedis.make_batch(); + for (const auto& dele_tuple : ranges) { + const auto set_id = std::get<0>(dele_tuple); + const auto file_base_index = std::get<1>(dele_tuple); + const auto file_count = std::get<2>(dele_tuple); + + auto hash_query_batch = parRedis.make_batch(); + for (FileIDType i = file_base_index; i < file_base_index + file_count; ++i) { + const auto file_key = PROGRAM_NAME ":file:" + lexical_cast(i); + hash_query_batch.run("HGET", file_key, "hash"); + } + hash_query_batch.throw_if_failed(); + + for (const auto& rep : hash_query_batch.replies()) { + const auto hash_key = PROGRAM_NAME ":hash:" + redis::get_string(rep); + parDeleHash.run( + dele_batch, + std::make_tuple(hash_key), + std::make_tuple(lexical_cast(file_base_index), lexical_cast(file_count)) + ); + } + + dele_batch.run("DEL", PROGRAM_NAME ":set:" + lexical_cast(set_id)); + chunked_run(dele_batch, +"DEL", file_base_index, file_count, [](FileIDType id){return PROGRAM_NAME ":file:" + lexical_cast(id);}); + } + + dele_batch.throw_if_failed(); + } +} //namespace dindb diff --git a/src/backends/redis/delete.hpp b/src/backends/redis/delete.hpp new file mode 100644 index 0000000..75074ac --- /dev/null +++ b/src/backends/redis/delete.hpp @@ -0,0 +1,45 @@ +/* Copyright 2015, 2016, Michele Santullo + * This file is part of "dindexer". + * + * "dindexer" is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * "dindexer" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "dindexer". If not, see . + */ + +#ifndef id1E0F8FEB88EC4FED843FFEE7BAB624BB +#define id1E0F8FEB88EC4FED843FFEE7BAB624BB + +#include "backends/db_backend.hpp" +#include +#include +#include +#include + +namespace redis { + class Command; + class Script; +} //namespace redis + +namespace dindb { + using IDDescMap = std::map; + using ConfirmDeleCallback = std::function; + + void delete_group_from_db ( + redis::Command& parRedis, + redis::Script& parDeleTagIfInSet, + redis::Script& parDeleHash, + const std::vector& parIDs, + ConfirmDeleCallback parConf + ); +} //namespace dindb + +#endif