King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-02-19 12:04:54 +00:00
Code Issues Releases Wiki Activity

Re-enable locate in the postgre plugin.

Browse source
This commit is contained in:
King_DuckZ 2016-06-01 10:27:09 +02:00
parent 470738fc1b
commit 780cd9b0ca
8 changed files with 80 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CMakeLists.txt
View file

@ -135,7 +135,7 @@ add_subdirectory(src/main)
add_subdirectory(src/scan)
add_subdirectory(src/delete)
#add_subdirectory(src/query)
#add_subdirectory(src/locate)
add_subdirectory(src/locate)
#add_subdirectory(src/navigate)
add_subdirectory(src/tag)

27
include/backends/db_backend.hpp
View file

@ -19,7 +19,6 @@
#define id7506CA9825454B80856154ACFE8A9DE2
#include "backends/backend_loader.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include <cstdint>
#include <string>
#include <vector>
@ -27,15 +26,36 @@
#include <map>
#include <functional>
namespace mchlib {
struct TigerHash;
struct FileRecordData;
struct SetRecordData;
struct SetRecordDataFull;
} //namespace mchlib
namespace dindb {
using GroupIDType = uint32_t;
using FileIDType = uint64_t;
using IDDescMap = std::map<uint32_t, std::string>;
using ConfirmDeleCallback = std::function<bool(const IDDescMap&)>;
using TagList = std::vector<boost::string_ref>;
constexpr const GroupIDType InvalidGroupID = 0;
constexpr const FileIDType InvalidFileID = 0;
struct LocatedItem {
std::string path;
FileIDType id;
GroupIDType group_id;
};
struct LocatedSet {
std::string desc;
GroupIDType id;
uint32_t files_count;
uint32_t dir_count;
};
class Backend {
public:
Backend ( void ) = default;
@ -52,6 +72,11 @@ namespace dindb {
virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature ) = 0;
virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash ) = 0;
virtual std::vector<LocatedItem> locate_in_db ( const std::string& parSearch, const TagList& parTags ) = 0;
virtual std::vector<LocatedItem> locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags ) = 0;
virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive ) = 0;
virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive ) = 0;
};
} //namespace dindb

2
src/backends/postgresql/CMakeLists.txt
View file

@ -3,7 +3,7 @@ project(${bare_name}-backend-postgresql CXX)
add_library(${PROJECT_NAME} SHARED
tag.cpp
delete.cpp
#locate.cpp
locate.cpp
scan.cpp
#dbsource.cpp
backend_postgresql.cpp

17
src/backends/postgresql/backend_postgresql.cpp
View file

@ -20,6 +20,7 @@
#include "tag.hpp"
#include "delete.hpp"
#include "scan.hpp"
#include "locate.hpp"
#include "pq/connection.hpp"
#include <ciso646>
#include <utility>
@ -113,6 +114,22 @@ namespace dindb {
bool BackendPostgreSql::search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash) {
return dindb::read_from_db(parItem, parSet, *m_conn, parHash);
}
std::vector<LocatedItem> BackendPostgreSql::locate_in_db (const std::string& parSearch, const TagList& parTags) {
return dindb::locate_in_db(*m_conn, parSearch, parTags);
}
std::vector<LocatedItem> BackendPostgreSql::locate_in_db (const mchlib::TigerHash& parSearch, const TagList& parTags) {
return dindb::locate_in_db(*m_conn, parSearch, parTags);
}
std::vector<LocatedSet> BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, bool parCaseInsensitive) {
return dindb::locate_sets_in_db(*m_conn, parSearch, parCaseInsensitive);
}
std::vector<LocatedSet> BackendPostgreSql::locate_sets_in_db (const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
return dindb::locate_sets_in_db(*m_conn, parSearch, parSets, parCaseInsensitive);
}
} //namespace dindb
extern "C" dindb::Backend* dindexer_create_backend (const YAML::Node* parConfig) {

5
src/backends/postgresql/backend_postgresql.hpp
View file

@ -46,6 +46,11 @@ namespace dindb {
virtual void write_files ( const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature );
virtual bool search_file_by_hash ( mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const mchlib::TigerHash& parHash );
virtual std::vector<LocatedItem> locate_in_db ( const std::string& parSearch, const TagList& parTags );
virtual std::vector<LocatedItem> locate_in_db ( const mchlib::TigerHash& parSearch, const TagList& parTags );
virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, bool parCaseInsensitive );
virtual std::vector<LocatedSet> locate_sets_in_db ( const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive );
private:
std::unique_ptr<pq::Connection> m_conn;
};

39
src/backends/postgresql/locate.cpp
View file

@ -15,28 +15,19 @@
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#include "db/locate.hpp"
#include "db/settings.hpp"
#include "locate.hpp"
#include "dindexer-machinery/recorddata.hpp"
#include "pq/connection.hpp"
#include "dindexer-machinery/tiger.hpp"
#include <utility>
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <cassert>
namespace dindb {
namespace {
const int g_max_results = 200;
pq::Connection make_pq_conn ( const Settings& parDB, bool parOpen=true );
pq::Connection make_pq_conn (const Settings& parDB, bool parOpen) {
auto conn = pq::Connection(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
if (parOpen) {
conn.connect();
}
return conn;
}
std::vector<LocatedSet> sets_result_to_vec (pq::ResultSet&& parResult) {
using boost::lexical_cast;
@ -95,21 +86,21 @@ namespace dindb {
}
} //unnamed namespace
std::vector<LocatedItem> locate_in_db (const Settings& parDB, const std::string& parSearch, const TagList& parTags) {
auto conn = make_pq_conn(parDB);
std::vector<LocatedItem> locate_in_db (pq::Connection& parDB, const std::string& parSearch, const TagList& parTags) {
assert(parDB.is_connected());
const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"path\" ~ $1";
return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch);
return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch);
}
std::vector<LocatedItem> locate_in_db (const Settings& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
auto conn = make_pq_conn(parDB);
std::vector<LocatedItem> locate_in_db (pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
assert(parDB.is_connected());
const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"hash\"=$1";
return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true));
return locate_in_db(parDB, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true));
}
std::vector<LocatedSet> locate_sets_in_db (const Settings& parDB, const std::string& parSearch, bool parCaseInsensitive) {
auto conn = make_pq_conn(parDB);
std::vector<LocatedSet> locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive) {
assert(parDB.is_connected());
const std::string query = std::string("SELECT \"id\", \"desc\", "
"(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", "
@ -117,16 +108,16 @@ namespace dindb {
"FROM \"sets\" WHERE str_match_partial(\"desc\", $1, $2) LIMIT "
) + std::to_string(g_max_results) + ";";
auto result = conn.query(query, parSearch, parCaseInsensitive);
auto result = parDB.query(query, parSearch, parCaseInsensitive);
return sets_result_to_vec(std::move(result));
}
std::vector<LocatedSet> locate_sets_in_db (const Settings& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive) {
std::vector<LocatedSet> locate_sets_in_db (pq::Connection& parDB, const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive) {
if (parSets.empty()) {
return locate_sets_in_db(parDB, parSearch, parCaseInsensitive);
}
auto conn = make_pq_conn(parDB);
assert(parDB.is_connected());
const std::string query = std::string("SELECT \"id\", \"desc\", "
"(SELECT COUNT(*) FROM \"files\" WHERE \"group_id\"=\"sets\".\"id\" AND NOT \"is_directory\") as \"file_count\", "
@ -134,7 +125,7 @@ namespace dindb {
"FROM \"sets\" WHERE \"id\" = ANY($1) AND str_match_partial(\"desc\", $3, $2) LIMIT "
) + std::to_string(g_max_results) + ";";
auto result = conn.query(query, parSearch, parCaseInsensitive, parSets);
auto result = parDB.query(query, parSearch, parCaseInsensitive, parSets);
return sets_result_to_vec(std::move(result));
}
} //namespace dindb

33
src/backends/postgresql/locate.hpp
View file

@ -21,34 +21,17 @@
#include <vector>
#include <string>
#include <cstdint>
#include <boost/utility/string_ref.hpp>
#include "backends/db_backend.hpp"
namespace mchlib {
struct TigerHash;
} //namespace mchlib
namespace pq {
class Connection;
} //namespace pq
namespace dindb {
struct Settings;
struct LocatedItem {
std::string path;
uint64_t id;
uint32_t group_id;
};
struct LocatedSet {
std::string desc;
uint32_t id;
uint32_t files_count;
uint32_t dir_count;
};
using TagList = std::vector<boost::string_ref>;
std::vector<LocatedItem> locate_in_db ( const Settings& parDB, const std::string& parSearch, const TagList& parTags );
std::vector<LocatedItem> locate_in_db ( const Settings& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags );
std::vector<LocatedSet> locate_sets_in_db ( const Settings& parDB, const std::string& parSearch, bool parCaseInsensitive );
std::vector<LocatedSet> locate_sets_in_db ( const Settings& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive );
std::vector<LocatedItem> locate_in_db ( pq::Connection& parDB, const std::string& parSearch, const TagList& parTags );
std::vector<LocatedItem> locate_in_db ( pq::Connection& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags );
std::vector<LocatedSet> locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, bool parCaseInsensitive );
std::vector<LocatedSet> locate_sets_in_db ( pq::Connection& parDB, const std::string& parSearch, const std::vector<GroupIDType>& parSets, bool parCaseInsensitive );
} //namespace dindb
#endif

11
src/locate/main.cpp
View file

@ -16,7 +16,6 @@
*/
#include "commandline.hpp"
#include "db/locate.hpp"
#include "dindexer-common/settings.hpp"
#include "dindexer-common/split_tags.hpp"
#include "dindexerConfig.h"
@ -79,9 +78,13 @@ int main (int parArgc, char* parArgv[]) {
return 1;
}
}
//TODO: throw if plugin loading failed
assert(settings.backend_plugin.name() == settings.backend_name);
assert(settings.backend_plugin.is_loaded());
auto& db = settings.backend_plugin.backend();
if (vm.count("set")) {
const auto results = dindb::locate_sets_in_db(settings.db, vm["substring"].as<std::string>(), not not vm.count("case-insensitive"));
const auto results = db.locate_sets_in_db(vm["substring"].as<std::string>(), not not vm.count("case-insensitive"));
std::copy(results.begin(), results.end(), std::ostream_iterator<dindb::LocatedSet>(std::cout, "\n"));
}
else {
@ -90,11 +93,11 @@ int main (int parArgc, char* parArgv[]) {
if (vm.count("byhash")) {
const auto hash = din::hash(vm["substring"].as<std::string>());
results = dindb::locate_in_db(settings.db, hash, tags);
results = db.locate_in_db(hash, tags);
}
else {
const auto search_regex = g2r::convert(vm["substring"].as<std::string>(), not vm.count("case-insensitive"));
results = dindb::locate_in_db(settings.db, search_regex, tags);
results = db.locate_in_db(search_regex, tags);
}
std::copy(results.begin(), results.end(), std::ostream_iterator<dindb::LocatedItem>(std::cout, "\n"));
}