mirror of
https://github.com/KingDuckZ/dindexer.git
synced 2025-07-13 15:54:11 +00:00
Move db dbsource files to the postgresql backend lib and rename namespace.
This commit is contained in:
parent
baa67138eb
commit
70caa9e26c
24 changed files with 166 additions and 131 deletions
|
|
@ -5,6 +5,7 @@ add_library(${PROJECT_NAME} STATIC
|
|||
delete.cpp
|
||||
locate.cpp
|
||||
scan.cpp
|
||||
dbsource.cpp
|
||||
)
|
||||
|
||||
#target_include_directories(${PROJECT_NAME}
|
||||
|
|
|
|||
183
src/backends/postgresql/dbsource.cpp
Normal file
183
src/backends/postgresql/dbsource.cpp
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
/* Copyright 2015, 2016, Michele Santullo
|
||||
* This file is part of "dindexer".
|
||||
*
|
||||
* "dindexer" is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* "dindexer" is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "backends/postgresql/dbsource.hpp"
|
||||
#include "backends/postgresql/settings.hpp"
|
||||
#include "pq/connection.hpp"
|
||||
#include "helpers/infix_iterator.hpp"
|
||||
#include <ciso646>
|
||||
#include <utility>
|
||||
#include <boost/lexical_cast.hpp>
|
||||
#include <sstream>
|
||||
#include <boost/range/algorithm/copy.hpp>
|
||||
|
||||
namespace dinbpostgres {
|
||||
namespace {
|
||||
const uint32_t g_files_query_limit = 500;
|
||||
|
||||
std::ostream& operator<< (std::ostream& parOut, const std::vector<std::string>& parCols) {
|
||||
parOut << '"';
|
||||
boost::copy(parCols, infix_ostream_iterator<std::string>(parOut, "\", \""));
|
||||
parOut << '"';
|
||||
return parOut;
|
||||
}
|
||||
} //unnamed namespace
|
||||
|
||||
const DBSource::SetDetailsMap DBSource::m_set_details_map {
|
||||
{SetDetail_Desc, "desc"},
|
||||
{SetDetail_Type, "type"},
|
||||
{SetDetail_CreeationDate, "creation"},
|
||||
{SetDetail_AppName, "app_name"},
|
||||
{SetDetail_ID, "id"}
|
||||
};
|
||||
const DBSource::FileDetailsMap DBSource::m_file_details_map {
|
||||
{FileDetail_ID, "id"},
|
||||
{FileDetail_Path, "path"},
|
||||
{FileDetail_Level, "level"},
|
||||
{FileDetail_GroupID, "group_id"},
|
||||
{FileDetail_IsDir, "is_directory"},
|
||||
{FileDetail_IsSymLink, "is_symlink"},
|
||||
{FileDetail_Size, "size"},
|
||||
{FileDetail_Hash, "hash"},
|
||||
{FileDetail_IsHashValid, "is_hash_valid"},
|
||||
{FileDetail_ATime, "access_time"},
|
||||
{FileDetail_MTime, "modify_time"},
|
||||
{FileDetail_Unreadable, "unreadable"},
|
||||
{FileDetail_MimeType, "mimetype"},
|
||||
{FileDetail_Charset, "charset"}
|
||||
};
|
||||
|
||||
struct DBSource::LocalData {
|
||||
explicit LocalData ( const Settings& parDBSettings ) :
|
||||
conn(
|
||||
std::string(parDBSettings.username),
|
||||
std::string(parDBSettings.password),
|
||||
std::string(parDBSettings.dbname),
|
||||
std::string(parDBSettings.address),
|
||||
parDBSettings.port
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
pq::Connection conn;
|
||||
};
|
||||
|
||||
DBSource::DBSource (const Settings& parDBSettings) :
|
||||
m_local_data(new LocalData(parDBSettings))
|
||||
{
|
||||
assert(not m_local_data->conn.is_connected());
|
||||
}
|
||||
|
||||
DBSource::~DBSource() noexcept {
|
||||
}
|
||||
|
||||
void DBSource::disconnect() {
|
||||
if (m_local_data->conn.is_connected()) {
|
||||
m_local_data->conn.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
pq::Connection& DBSource::get_conn() {
|
||||
if (not m_local_data->conn.is_connected()) {
|
||||
m_local_data->conn.connect();
|
||||
}
|
||||
return m_local_data->conn;
|
||||
}
|
||||
|
||||
std::vector<uint32_t> DBSource::sets() {
|
||||
using boost::lexical_cast;
|
||||
|
||||
auto& conn = get_conn();
|
||||
const std::string query = "SELECT \"id\" FROM \"sets\";";
|
||||
auto res = conn.query(query);
|
||||
std::vector<uint32_t> retval;
|
||||
|
||||
retval.reserve(res.size());
|
||||
for (const auto& row : res) {
|
||||
retval.push_back(lexical_cast<uint32_t>(row[0]));
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
|
||||
void DBSource::query_no_conditions (const ColumnList& parCols, boost::string_ref parTable, const std::vector<uint32_t>& parIDs, std::function<void(std::string&&)> parCallback) {
|
||||
std::ostringstream oss;
|
||||
oss << "SELECT " << parCols << ' ' <<
|
||||
"FROM \"" << parTable << "\" " <<
|
||||
"WHERE \"id\"=ANY($1) " <<
|
||||
"ORDER BY \"desc\" ASC " <<
|
||||
"LIMIT " << g_files_query_limit << ';';
|
||||
|
||||
auto& conn = get_conn();
|
||||
auto result = conn.query(oss.str(), parIDs);
|
||||
for (auto row : result) {
|
||||
for (auto val : row) {
|
||||
parCallback(std::move(val));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DBSource::query_files_in_dir (const ColumnList& parCols, boost::string_ref parDir, uint16_t parLevel, uint32_t parGroupID, QueryCallback parCallback) {
|
||||
std::ostringstream oss;
|
||||
oss << "SELECT " << parCols << ' ' <<
|
||||
"FROM \"files\" WHERE " <<
|
||||
"\"level\"=$1 " <<
|
||||
"AND \"group_id\"=$2 " <<
|
||||
"AND str_begins_with(\"path\", COALESCE($3, '')) " <<
|
||||
"ORDER BY \"is_directory\" DESC, \"path\" ASC " <<
|
||||
"LIMIT " << g_files_query_limit << ';';
|
||||
|
||||
auto& conn = get_conn();
|
||||
auto result = conn.query(
|
||||
oss.str(),
|
||||
parLevel,
|
||||
parGroupID,
|
||||
parDir
|
||||
);
|
||||
for (auto row : result) {
|
||||
for (auto val : row) {
|
||||
parCallback(std::move(val));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> DBSource::paths_starting_by (uint32_t parGroupID, uint16_t parLevel, boost::string_ref parPath) {
|
||||
std::ostringstream oss;
|
||||
oss << "SELECT \"path\" ||\n" <<
|
||||
"(SELECT CASE \"is_directory\"\n" <<
|
||||
"WHEN TRUE THEN '/'\n" <<
|
||||
"ELSE ''\n" <<
|
||||
"END) as path FROM \"files\" WHERE \"group_id\"=$1 AND " <<
|
||||
"\"level\"=$2 AND str_begins_with(\"path\", COALESCE($3, '')) " <<
|
||||
"ORDER BY \"is_directory\" DESC, \"path\" ASC LIMIT " <<
|
||||
g_files_query_limit << ';';
|
||||
|
||||
auto& conn = get_conn();
|
||||
auto result = conn.query(
|
||||
oss.str(),
|
||||
parGroupID,
|
||||
parLevel,
|
||||
parPath
|
||||
);
|
||||
std::vector<std::string> retval;
|
||||
retval.reserve(retval.size());
|
||||
for (auto row : result) {
|
||||
assert(not row.empty());
|
||||
retval.push_back(row[0]);
|
||||
}
|
||||
return retval;
|
||||
}
|
||||
} //namespace dinbpostgres
|
||||
|
|
@ -16,8 +16,8 @@
|
|||
*/
|
||||
|
||||
#include "backends/postgresql/delete.hpp"
|
||||
#include "backends/postgresql/settings.hpp"
|
||||
#include "pq/connection.hpp"
|
||||
#include "dindexer-common/settings.hpp"
|
||||
#include "helpers/infix_iterator.hpp"
|
||||
#include <sstream>
|
||||
#include <utility>
|
||||
|
|
@ -27,7 +27,7 @@
|
|||
#include <boost/range/algorithm/copy.hpp>
|
||||
#include <boost/lexical_cast.hpp>
|
||||
|
||||
namespace din {
|
||||
namespace dinbpostgres {
|
||||
namespace {
|
||||
IDDescMap fetch_existing_ids (pq::Connection& parConn, const std::vector<uint32_t>& parIDs) {
|
||||
using boost::lexical_cast;
|
||||
|
|
@ -50,7 +50,7 @@ namespace din {
|
|||
}
|
||||
} //unnamed namespace
|
||||
|
||||
void delete_group_from_db (const dinlib::SettingsDB& parDB, const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
|
||||
void delete_group_from_db (const Settings& parDB, const std::vector<uint32_t>& parIDs, ConfirmDeleCallback parConf) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
const auto dele_ids = fetch_existing_ids(conn, parIDs);
|
||||
|
|
@ -72,4 +72,4 @@ namespace din {
|
|||
|
||||
conn.query(oss.str());
|
||||
}
|
||||
} //namespace din
|
||||
} //namespace dinbpostgres
|
||||
|
|
|
|||
|
|
@ -16,19 +16,20 @@
|
|||
*/
|
||||
|
||||
#include "backends/postgresql/locate.hpp"
|
||||
#include "backends/postgresql/settings.hpp"
|
||||
#include "pq/connection.hpp"
|
||||
#include "dindexer-machinery/tiger.hpp"
|
||||
#include <utility>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/lexical_cast.hpp>
|
||||
|
||||
namespace din {
|
||||
namespace dinbpostgres {
|
||||
namespace {
|
||||
const int g_max_results = 200;
|
||||
|
||||
pq::Connection make_pq_conn ( const dinlib::SettingsDB& parDB, bool parOpen=true );
|
||||
pq::Connection make_pq_conn ( const Settings& parDB, bool parOpen=true );
|
||||
|
||||
pq::Connection make_pq_conn (const dinlib::SettingsDB& parDB, bool parOpen) {
|
||||
pq::Connection make_pq_conn (const Settings& parDB, bool parOpen) {
|
||||
auto conn = pq::Connection(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
if (parOpen) {
|
||||
conn.connect();
|
||||
|
|
@ -94,20 +95,20 @@ namespace din {
|
|||
}
|
||||
} //unnamed namespace
|
||||
|
||||
std::vector<LocatedItem> locate_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const TagList& parTags) {
|
||||
std::vector<LocatedItem> locate_in_db (const Settings& parDB, const std::string& parSearch, const TagList& parTags) {
|
||||
auto conn = make_pq_conn(parDB);
|
||||
|
||||
const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"path\" ~ $1";
|
||||
return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, parSearch);
|
||||
}
|
||||
|
||||
std::vector<LocatedItem> locate_in_db (const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
|
||||
std::vector<LocatedItem> locate_in_db (const Settings& parDB, const mchlib::TigerHash& parSearch, const TagList& parTags) {
|
||||
auto conn = make_pq_conn(parDB);
|
||||
const char base_query[] = "SELECT \"path\",\"id\",\"group_id\" FROM \"files\" WHERE \"hash\"=$1";
|
||||
return locate_in_db(conn, base_query, sizeof(base_query) - 1, "$2", parTags, mchlib::tiger_to_string(parSearch, true));
|
||||
}
|
||||
|
||||
std::vector<LocatedSet> locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, bool parCaseInsensitive) {
|
||||
std::vector<LocatedSet> locate_sets_in_db (const Settings& parDB, const std::string& parSearch, bool parCaseInsensitive) {
|
||||
auto conn = make_pq_conn(parDB);
|
||||
|
||||
const std::string query = std::string("SELECT \"id\", \"desc\", "
|
||||
|
|
@ -120,7 +121,7 @@ namespace din {
|
|||
return sets_result_to_vec(std::move(result));
|
||||
}
|
||||
|
||||
std::vector<LocatedSet> locate_sets_in_db (const dinlib::SettingsDB& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive) {
|
||||
std::vector<LocatedSet> locate_sets_in_db (const Settings& parDB, const std::string& parSearch, const std::vector<uint32_t>& parSets, bool parCaseInsensitive) {
|
||||
if (parSets.empty()) {
|
||||
return locate_sets_in_db(parDB, parSearch, parCaseInsensitive);
|
||||
}
|
||||
|
|
@ -136,4 +137,4 @@ namespace din {
|
|||
auto result = conn.query(query, parSearch, parCaseInsensitive, parSets);
|
||||
return sets_result_to_vec(std::move(result));
|
||||
}
|
||||
} //namespace din
|
||||
} //namespace dinbpostgres
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@
|
|||
*/
|
||||
|
||||
#include "backends/postgresql/scan.hpp"
|
||||
#include "backends/postgresql/settings.hpp"
|
||||
#include "pq/connection.hpp"
|
||||
#include "dindexer-common/settings.hpp"
|
||||
#include "dindexer-machinery/recorddata.hpp"
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
|
@ -28,11 +28,11 @@
|
|||
#include <boost/utility/string_ref.hpp>
|
||||
#include <chrono>
|
||||
|
||||
namespace din {
|
||||
namespace dinbpostgres {
|
||||
namespace {
|
||||
} //unnamed namespace
|
||||
|
||||
bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const mchlib::TigerHash& parHash) {
|
||||
bool read_from_db (mchlib::FileRecordData& parItem, mchlib::SetRecordDataFull& parSet, const Settings& parDB, const mchlib::TigerHash& parHash) {
|
||||
using boost::lexical_cast;
|
||||
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
|
|
@ -82,7 +82,7 @@ namespace din {
|
|||
return true;
|
||||
}
|
||||
|
||||
void write_to_db (const dinlib::SettingsDB& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
|
||||
void write_to_db (const Settings& parDB, const std::vector<mchlib::FileRecordData>& parData, const mchlib::SetRecordData& parSetData, const std::string& parSignature) {
|
||||
using std::chrono::system_clock;
|
||||
using boost::lexical_cast;
|
||||
|
||||
|
|
@ -144,4 +144,4 @@ namespace din {
|
|||
}
|
||||
conn.query("COMMIT;");
|
||||
}
|
||||
} //namespace din
|
||||
} //namespace dinbpostgres
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@
|
|||
*/
|
||||
|
||||
#include "backends/postgresql/tag.hpp"
|
||||
#include "backends/postgresql/settings.hpp"
|
||||
#include "pq/connection.hpp"
|
||||
#include "dindexer-common/settings.hpp"
|
||||
#include <ciso646>
|
||||
|
||||
namespace din {
|
||||
void tag_files (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
namespace dinbpostgres {
|
||||
void tag_files (const Settings& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -37,7 +38,7 @@ namespace din {
|
|||
}
|
||||
}
|
||||
|
||||
void tag_files (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
void tag_files (const Settings& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -71,7 +72,7 @@ namespace din {
|
|||
}
|
||||
}
|
||||
|
||||
void delete_tags (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
void delete_tags (const Settings& parDB, const std::vector<uint64_t>& parFiles, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -97,7 +98,7 @@ namespace din {
|
|||
}
|
||||
}
|
||||
|
||||
void delete_tags (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
void delete_tags (const Settings& parDB, const std::vector<std::string>& parRegexes, const std::vector<boost::string_ref>& parTags, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -123,7 +124,7 @@ namespace din {
|
|||
}
|
||||
}
|
||||
|
||||
void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector<uint64_t>& parFiles, OwnerSetInfo parSet) {
|
||||
void delete_all_tags (const Settings& parDB, const std::vector<uint64_t>& parFiles, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -139,7 +140,7 @@ namespace din {
|
|||
}
|
||||
}
|
||||
|
||||
void delete_all_tags (const dinlib::SettingsDB& parDB, const std::vector<std::string>& parRegexes, OwnerSetInfo parSet) {
|
||||
void delete_all_tags (const Settings& parDB, const std::vector<std::string>& parRegexes, OwnerSetInfo parSet) {
|
||||
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
|
||||
conn.connect();
|
||||
|
||||
|
|
@ -152,4 +153,4 @@ namespace din {
|
|||
conn.query(query, parRegexes);
|
||||
}
|
||||
}
|
||||
} //namespace din
|
||||
} //namespace dinbpostgres
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue