Implement delete_group.

src/backends/redis/CMakeLists.txt

@@ -17,6 +17,7 @@ add_library(${PROJECT_NAME} SHARED
 	script_manager.cpp
 	async_connection.cpp
 	tag.cpp
+	delete.cpp
 )
 
 target_include_directories(${PROJECT_NAME} SYSTEM
@@ -57,6 +58,7 @@ configure_file(
 set(LUA_SCRIPTS
 	tag_if_in_set.lua
 	dele_tag_if_in_set.lua
+	dele_hash.lua
 )
 
 install(TARGETS ${PROJECT_NAME}

src/backends/redis/backend_redis.cpp

@@ -23,6 +23,7 @@
 #include "dindexerConfig.h"
 #include "helpers/stringize.h"
 #include "tag.hpp"
+#include "delete.hpp"
 #include "record_data_adapt.hpp"
 #include <utility>
 #include <yaml-cpp/yaml.h>
@@ -108,6 +109,7 @@ namespace dindb {
 			auto batch = m_redis.make_batch();
 			batch.run("SELECT", lexical_cast<std::string>(m_database));
 			batch.run("CLIENT", "SETNAME", PROGRAM_NAME "_v" STRINGIZE(VERSION_MAJOR) "." STRINGIZE(VERSION_MINOR) "." STRINGIZE(VERSION_PATCH));
+			batch.run("SCRIPT", "FLUSH");
 			batch.throw_if_failed();
 		}
 		else {
@@ -118,6 +120,7 @@ namespace dindb {
 
 		m_tag_if_in_set = m_redis.make_script(read_script(m_lua_script_paths, "tag_if_in_set.lua"));
 		m_dele_tag_if_in_set = m_redis.make_script(read_script(m_lua_script_paths, "dele_tag_if_in_set.lua"));
+		m_dele_hash = m_redis.make_script(read_script(m_lua_script_paths, "dele_hash.lua"));
 	}
 
 	void BackendRedis::disconnect() {
@@ -149,6 +152,7 @@ namespace dindb {
 	}
 
 	void BackendRedis::delete_group (const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+		delete_group_from_db(m_redis, m_dele_tag_if_in_set, m_dele_hash, parIDs, parConf);
 	}
 
 	void BackendRedis::write_files (const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
@@ -173,7 +177,9 @@ namespace dindb {
 			"disk_label", parSetData.disk_label,
 			"fs_uuid", parSetData.fs_uuid,
 			"type", parSetData.type,
-			"content_type", parSetData.content_type
+			"content_type", parSetData.content_type,
+			"base_file_id", lexical_cast<std::string>(base_file_id),
+			"file_count", lexical_cast<std::string>(parData.size())
 		);
 
 		for (auto z = base_file_id; z < casted_data_size + 1; ++z) {

src/backends/redis/backend_redis.hpp

@@ -61,6 +61,7 @@ namespace dindb {
 		redis::Command m_redis;
 		redis::Script m_tag_if_in_set;
 		redis::Script m_dele_tag_if_in_set;
+		redis::Script m_dele_hash;
 		dincore::SearchPaths m_lua_script_paths;
 		uint16_t m_database;
 	};

src/backends/redis/dele_hash.lua

@@ -0,0 +1,27 @@
+-- Copyright 2015, 2016, Michele Santullo
+-- This file is part of "dindexer".
+--
+-- "dindexer" is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License as published by
+-- the Free Software Foundation, either version 3 of the License, or
+-- (at your option) any later version.
+--
+-- "dindexer" is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+
+local hash_key = KEYS[1]
+local base_index = ARGV[1]
+local file_count = ARGV[2]
+
+for z = base_index, base_index + file_count - 1 do
+	redis.call("SREM", hash_key, z)
+end
+if redis.call("SCARD", hash_key) == 0 then
+	redis.call("DEL", hash_key)
+end
+return nil

src/backends/redis/delete.cpp

@@ -0,0 +1,152 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "delete.hpp"
+#include "tag.hpp"
+#include "command.hpp"
+#include "helpers/lexical_cast.hpp"
+#include "helpers/sequence_bt.hpp"
+#include "dindexerConfig.h"
+#include <vector>
+#include <tuple>
+#include <utility>
+#include <boost/iterator/counting_iterator.hpp>
+#include <algorithm>
+
+namespace dindb {
+	namespace {
+		std::pair<bool, std::size_t> confirm_dele (redis::Batch& parBatch, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+			using dinhelp::lexical_cast;
+
+			if (parIDs.empty())
+				return std::make_pair(false, parIDs.size());
+
+			for (auto id : parIDs) {
+				const auto set_key = PROGRAM_NAME ":set:" + lexical_cast<std::string>(id);
+				parBatch.run("HMGET", set_key, "base_file_id", "file_count", "name");
+			}
+
+			std::map<GroupIDType, std::string> set_dele_list;
+			std::size_t id_index = 0;
+			for (const auto& reply : parBatch.replies()) {
+				const auto res = redis::get_array(reply);
+				assert(res.size() == 3);
+				if (redis::RedisVariantType_Nil != res[0].which()) {
+					assert(redis::RedisVariantType_Nil != res[1].which());
+					set_dele_list[parIDs[id_index]] = redis::get_string(res[2]);
+				}
+				++id_index;
+			}
+
+			if (set_dele_list.empty())
+				return std::make_pair(false, set_dele_list.size());
+			else
+				return std::make_pair(parConf(set_dele_list), set_dele_list.size());
+				//return std::make_pair(true, set_dele_list.size());
+		}
+
+		template <typename IT, IT CHUNK, typename F, IT... SVALS>
+		void chunked_run_pvt (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey, dinhelp::bt::number_seq<IT, SVALS...>) {
+			for (IT i = 0; i < parCount / CHUNK; ++i) {
+				parBatch.run(parCommand, parMakeKey(i * CHUNK + parFrom + SVALS)...);
+			}
+			for (auto i = ((parFrom + parCount) / CHUNK) * CHUNK; i < parFrom + parCount; ++i) {
+				const auto key = parMakeKey(i);
+				parBatch.run(parCommand, key);
+			}
+		}
+
+		template <typename IT, IT CHUNK, typename F>
+		void chunked_run (redis::Batch& parBatch, const char* parCommand, IT parFrom, IT parCount, F parMakeKey) {
+			chunked_run_pvt<IT, CHUNK, F>(parBatch, parCommand, parFrom, parCount, parMakeKey, dinhelp::bt::number_range<IT, 0, CHUNK>());
+		};
+	} //unnamed namespace
+
+	void delete_group_from_db (redis::Command& parRedis, redis::Script& parDeleTagIfInSet, redis::Script& parDeleHash, const std::vector<GroupIDType>& parIDs, ConfirmDeleCallback parConf) {
+		using dinhelp::lexical_cast;
+		using IDRange = std::tuple<GroupIDType, FileIDType, FileIDType>;
+
+		auto set_batch = parRedis.make_batch();
+
+		auto dele_pair = confirm_dele(set_batch, parIDs, parConf);
+		assert(set_batch.replies_requested());
+		if (not dele_pair.first)
+			return;
+
+		std::vector<IDRange> ranges;
+		ranges.reserve(dele_pair.second);
+		std::size_t id_index = 0;
+		for (const auto& reply : set_batch.replies()) {
+			const auto res = redis::get_array(reply);
+			if (redis::RedisVariantType_Nil != res[0].which()) {
+				ranges.push_back(
+					IDRange(
+						parIDs[id_index],
+						lexical_cast<FileIDType>(redis::get_string(res[0])),
+						lexical_cast<FileIDType>(redis::get_string(res[1]))
+					)
+				);
+			}
+			++id_index;
+		}
+		assert(ranges.size() == dele_pair.second);
+
+		for (const auto& dele_tuple : ranges) {
+			const auto set_id = std::get<0>(dele_tuple);
+			const auto file_base_index = std::get<1>(dele_tuple);
+			const auto file_count = std::get<2>(dele_tuple);
+
+			std::vector<FileIDType> ids;
+			ids.reserve(file_count);
+			std::copy(
+				boost::counting_iterator<FileIDType>(file_base_index),
+				boost::counting_iterator<FileIDType>(file_base_index + file_count),
+				std::back_inserter(ids)
+			);
+
+			delete_all_tags(parRedis, parDeleTagIfInSet, ids, set_id);
+		}
+
+		auto dele_batch = parRedis.make_batch();
+		for (const auto& dele_tuple : ranges) {
+			const auto set_id = std::get<0>(dele_tuple);
+			const auto file_base_index = std::get<1>(dele_tuple);
+			const auto file_count = std::get<2>(dele_tuple);
+
+			auto hash_query_batch = parRedis.make_batch();
+			for (FileIDType i = file_base_index; i < file_base_index + file_count; ++i) {
+				const auto file_key = PROGRAM_NAME ":file:" + lexical_cast<std::string>(i);
+				hash_query_batch.run("HGET", file_key, "hash");
+			}
+			hash_query_batch.throw_if_failed();
+
+			for (const auto& rep : hash_query_batch.replies()) {
+				const auto hash_key = PROGRAM_NAME ":hash:" + redis::get_string(rep);
+				parDeleHash.run(
+					dele_batch,
+					std::make_tuple(hash_key),
+					std::make_tuple(lexical_cast<std::string>(file_base_index), lexical_cast<std::string>(file_count))
+				);
+			}
+
+			dele_batch.run("DEL", PROGRAM_NAME ":set:" + lexical_cast<std::string>(set_id));
+			chunked_run<FileIDType, 8>(dele_batch, +"DEL", file_base_index, file_count, [](FileIDType id){return PROGRAM_NAME ":file:" + lexical_cast<std::string>(id);});
+		}
+
+		dele_batch.throw_if_failed();
+	}
+} //namespace dindb

src/backends/redis/delete.hpp

@@ -0,0 +1,45 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id1E0F8FEB88EC4FED843FFEE7BAB624BB
+#define id1E0F8FEB88EC4FED843FFEE7BAB624BB
+
+#include "backends/db_backend.hpp"
+#include <functional>
+#include <vector>
+#include <string>
+#include <map>
+
+namespace redis {
+	class Command;
+	class Script;
+} //namespace redis
+
+namespace dindb {
+	using IDDescMap = std::map<GroupIDType, std::string>;
+	using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
+
+	void delete_group_from_db (
+		redis::Command& parRedis,
+		redis::Script& parDeleTagIfInSet,
+		redis::Script& parDeleHash,
+		const std::vector<GroupIDType>& parIDs,
+		ConfirmDeleCallback parConf
+	);
+} //namespace dindb
+
+#endif