/* Copyright 2015, 2016, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see .
*/
#include "backend_redis.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include "backends/exposed_functions.hpp"
#include "backends/backend_version.hpp"
#include "helpers/lexical_cast.hpp"
#include "dindexerConfig.h"
#include "helpers/stringize.h"
#include "tag.hpp"
#include "record_data_adapt.hpp"
#include
#include
#include
#include
#include
#include
namespace dindb {
namespace {
struct RedisConnectionSettings {
std::string address;
uint16_t port;
uint16_t database;
};
std::string read_script (const dincore::SearchPaths& parSearch, const char* parName) {
const auto full_path = parSearch.first_hit(boost::string_ref(parName));
if (full_path.empty()) {
const std::string msg = std::string("Unable to locate and load Lua script \"") + parName + "\" from any of the given search paths";
throw std::runtime_error(msg);
}
std::ifstream script(full_path);
std::string retval;
script.seekg(0, std::ios::end);
retval.reserve(script.tellg());
script.seekg(0, std::ios::beg);
retval.assign(std::istreambuf_iterator(script), std::istreambuf_iterator());
return retval;
}
} //unnamed namespace
} //namespace dindb
namespace YAML {
template<>
struct convert {
static Node encode (const dindb::RedisConnectionSettings& parSettings) {
Node node;
node["address"] = parSettings.address;
node["port"] = parSettings.port;
node["database"] = parSettings.database;
return node;
}
static bool decode (const Node& parNode, dindb::RedisConnectionSettings& parSettings) {
if (not parNode.IsMap() or parNode.size() != 2) {
return false;
}
parSettings.address = parNode["address"].as();
parSettings.port = parNode["port"].as();
if (parNode["database"])
parSettings.database = parNode["database"].as();
else
parSettings.database = 0;
return true;
}
};
} //namespace YAML
namespace dindb {
BackendRedis::BackendRedis(std::string&& parAddress, uint16_t parPort, uint16_t parDatabase, bool parConnect, dincore::SearchPaths&& parLuaPaths) :
m_redis(std::move(parAddress), parPort),
m_tag_if_in_set(),
m_lua_script_paths(std::move(parLuaPaths)),
m_database(parDatabase)
{
if (parConnect)
this->connect();
}
BackendRedis::~BackendRedis() noexcept {
}
void BackendRedis::connect() {
using dinhelp::lexical_cast;
m_redis.connect();
m_redis.wait_for_connect();
if (m_redis.is_connected()) {
auto batch = m_redis.make_batch();
batch.run("SELECT", lexical_cast(m_database));
batch.run("CLIENT", "SETNAME", PROGRAM_NAME "_v" STRINGIZE(VERSION_MAJOR) "." STRINGIZE(VERSION_MINOR) "." STRINGIZE(VERSION_PATCH));
batch.throw_if_failed();
}
else {
std::ostringstream oss;
oss << "Error connecting to Redis: " << m_redis.connection_error();
throw std::runtime_error(oss.str());
}
m_tag_if_in_set = m_redis.make_script(read_script(m_lua_script_paths, "tag_if_in_set.lua"));
}
void BackendRedis::disconnect() {
m_redis.disconnect();
}
void BackendRedis::tag_files (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) {
dindb::tag_files(m_redis, m_tag_if_in_set, parFiles, parTags, parSet);
}
void BackendRedis::tag_files (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) {
dindb::tag_files(m_redis, m_tag_if_in_set, parRegexes, parTags, parSet);
}
void BackendRedis::delete_tags (const std::vector& parFiles, const std::vector& parTags, GroupIDType parSet) {
}
void BackendRedis::delete_tags (const std::vector& parRegexes, const std::vector& parTags, GroupIDType parSet) {
}
void BackendRedis::delete_all_tags (const std::vector& parFiles, GroupIDType parSet) {
}
void BackendRedis::delete_all_tags (const std::vector& parRegexes, GroupIDType parSet) {
}
void BackendRedis::delete_group (const std::vector& parIDs, ConfirmDeleCallback parConf) {
}
void BackendRedis::write_files (const std::vector& parData, const mchlib::SetRecordDataFull& parSetData, const std::string& parSignature) {
using dinhelp::lexical_cast;
using boost::string_ref;
redis::Reply set_id_reply = m_redis.run("HINCRBY", PROGRAM_NAME ":indices", "set", "1");
redis::Reply file_id_reply = m_redis.run("HINCRBY", PROGRAM_NAME ":indices", "files", lexical_cast(parData.size()));
const std::string set_key = PROGRAM_NAME ":set:" + lexical_cast(redis::get_integer(set_id_reply));
const auto casted_data_size = static_cast(parData.size());
assert(redis::get_integer(file_id_reply) >= casted_data_size);
const auto base_file_id = redis::get_integer(file_id_reply) - casted_data_size + 1;
auto batch = m_redis.make_batch();
batch.run(
"HMSET",
set_key,
"name", parSetData.name,
"disk_label", parSetData.disk_label,
"fs_uuid", parSetData.fs_uuid,
"type", parSetData.type,
"content_type", parSetData.content_type
);
for (auto z = base_file_id; z < casted_data_size; ++z) {
const std::string file_key = PROGRAM_NAME ":file:" + lexical_cast(z);
const auto& file_data = parData[z - base_file_id];
const std::string hash = tiger_to_string(file_data.hash);
batch.run(
"HMSET",
file_key,
"hash", hash,
"path", file_data.path(),
"size", lexical_cast(file_data.size),
"level", lexical_cast(file_data.level),
"mime_type", file_data.mime_type(),
"mime_charset", file_data.mime_charset(),
"is_directory", (file_data.is_directory ? '1' : '0'),
"is_symlink", (file_data.is_symlink ? '1' : '0'),
"unreadable", (file_data.unreadable ? '1' : '0'),
"hash_valid", (file_data.hash_valid ? '1' : '0'),
"group_id", set_key
);
batch.run(
"SADD",
PROGRAM_NAME ":hash:" + hash,
file_key
);
}
batch.throw_if_failed();
}
bool BackendRedis::search_file_by_hash (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
using boost::empty;
const std::string hash_key = PROGRAM_NAME ":hash:" + tiger_to_string(parHash);
redis::Reply hash_reply = m_redis.run("SRANDMEMBER", hash_key);
if (redis::RedisVariantType_Nil == hash_reply.which() or (redis::RedisVariantType_Integer == hash_reply.which() and not redis::get_integer(hash_reply))) {
return false;
}
else {
const auto result_id = redis::get_string(hash_reply);
auto set_key_and_file_item = redis::range_as(m_redis.hscan(result_id));
parItem = std::move(set_key_and_file_item.second);
const std::string group_key = std::move(set_key_and_file_item.first);
auto scan_range = m_redis.hscan(group_key);
if (empty(scan_range)) {
return false;
}
else {
parSet = redis::range_as(m_redis.hscan(group_key));
return true;
}
}
}
std::vector BackendRedis::locate_in_db (const std::string& parSearch, const TagList& parTags) {
return std::vector();
}
std::vector BackendRedis::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) {
return std::vector();
}
std::vector BackendRedis::locate_sets_in_db (const std::string& parSearch, bool parCaseInsensitive) {
return std::vector();
}
std::vector BackendRedis::locate_sets_in_db (const std::string& parSearch, const std::vector& parSets, bool parCaseInsensitive) {
return std::vector();
}
std::vector BackendRedis::find_all_sets() {
return std::vector();
}
std::vector> BackendRedis::find_set_details (const std::vector& parSets) {
return std::vector>();
}
std::vector> BackendRedis::find_file_details (GroupIDType parSetID, uint16_t parLevel, boost::string_ref parDir) {
return std::vector>();
}
std::vector BackendRedis::find_paths_starting_by (GroupIDType parGroupID, uint16_t parLevel, boost::string_ref parPath) {
return std::vector();
}
} //namespace dindb
extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {
if (not parConfig)
return nullptr;
auto& config_node = *parConfig;
auto config = config_node["connection"].as();
auto vec = (config_node["script_paths"] ? config_node["script_paths"].as>() : std::vector());
dincore::SearchPaths lua_paths(std::move(vec));
lua_paths.add_path(REDIS_SCRIPTS_PATH);
return new dindb::BackendRedis(
std::move(config.address),
config.port,
config.database,
true,
std::move(lua_paths)
);
}
extern "C" void dindexer_destroy_backend (dindb::Backend* parDele) {
if (parDele)
delete parDele;
}
extern "C" const char* dindexer_backend_name() {
return "redis";
}
extern "C" int dindexer_backend_iface_version() {
return dindb::g_current_iface_version;
}