King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-08-16 15:09:46 +00:00
Code Issues Releases Wiki Activity

Use parametric sql functions to insert new files.

Browse source 
Refactoring so that there are no extra copies of data being inserted.
This commit is contained in:
King_DuckZ 2015-12-29 17:32:22 +00:00
parent a91e75829f
commit 390b69e150
7 changed files with 360 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

67
src/scan/dbbackend.cpp
View file

@ -18,6 +18,7 @@
#include "dbbackend.hpp"
#include "pq/connection.hpp"
#include "dindexer-common/settings.hpp"
#include "recorddata.hpp"
#include <string>
#include <sstream>
#include <utility>
@ -25,11 +26,10 @@
#include <exception>
#include <memory>
#include <boost/utility/string_ref.hpp>
#include <chrono>
namespace din {
namespace {
const std::size_t g_batch_size = 100;
std::string make_set_insert_query (pq::Connection& parConn, const SetRecordData& parSetData) {
std::ostringstream oss;
oss << "INSERT INTO \"sets\" (\"desc\",\"type\") VALUES ("
@ -38,15 +38,9 @@ namespace din {
<< ");";
return oss.str();
}
boost::string_ref time_to_str (const std::time_t parTime, char* parBuff, std::size_t parLength) {
const auto gtm = std::gmtime(&parTime);
const auto len = std::strftime(parBuff, parLength, "%F %T%z", gtm);
return boost::string_ref(parBuff, len);
}
} //unnamed namespace
bool read_from_db (FileRecordData& parItem, SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, std::string&& parHash) {
bool read_from_db (FileRecordData& parItem, SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const TigerHash& parHash) {
using boost::lexical_cast;
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
@ -55,8 +49,8 @@ namespace din {
uint32_t group_id;
{
std::ostringstream oss;
oss << "SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash=" <<
conn.escaped_literal(parHash) <<
oss << "SELECT path,level,group_id,is_directory,is_symlink,size FROM files WHERE hash='" <<
tiger_to_string(parHash, true) << "'" <<
" LIMIT 1;";
auto resultset = conn.query(oss.str());
@ -66,7 +60,7 @@ namespace din {
auto row = resultset[0];
parItem.path = row["path"];
parItem.hash = std::move(parHash);
parItem.hash = parHash;
parItem.level = lexical_cast<uint16_t>(row["level"]);
parItem.size = lexical_cast<uint64_t>(row["size"]);
parItem.is_directory = (row["is_directory"] == "t" ? true : false);
@ -94,47 +88,36 @@ namespace din {
}
void write_to_db (const dinlib::SettingsDB& parDB, const std::vector<FileRecordData>& parData, const SetRecordData& parSetData) {
auto bool_to_str = [](bool b) { return (b ? "true" : "false"); };
using std::chrono::system_clock;
if (parData.empty()) {
return;
}
const std::size_t strtime_buff_size = 512;
std::unique_ptr<char[]> strtime_buff(new char[strtime_buff_size]);
pq::Connection conn(std::string(parDB.username), std::string(parDB.password), std::string(parDB.dbname), std::string(parDB.address), parDB.port);
conn.connect();
conn.query_void("BEGIN;");
conn.query_void(make_set_insert_query(conn, parSetData));
//TODO: use COPY instead of INSERT INTO
for (std::size_t z = 0; z < parData.size(); z += g_batch_size) {
std::ostringstream query;
query << "INSERT INTO \"files\" " <<
"(path, hash, level, group_id, is_directory, is_symlink, size, " <<
for (std::size_t z = 0; z < parData.size(); ++z) {
const std::string query = "INSERT INTO \"files\" (path, hash, "
"level, group_id, is_directory, is_symlink, size, "
"access_time, modify_time, is_hash_valid, unreadable) VALUES "
;
"($1, $2, $3, currval('\"sets_id_seq\"'), $4, $5, $6, $7, $8, $9, $10);";
const char* comma = "";
for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) {
const auto& itm = parData[i];
query << comma;
query << '(' << conn.escaped_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ','
<< "currval('\"sets_id_seq\"')" << ','
<< bool_to_str(itm.is_directory) << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size
<< ',' << '\'' << time_to_str(itm.atime, strtime_buff.get(), strtime_buff_size) << '\''
<< ',' << '\'' << time_to_str(itm.mtime, strtime_buff.get(), strtime_buff_size) << '\''
<< ',' << bool_to_str(itm.hash_valid)
<< ',' << bool_to_str(itm.unreadable)
<< ')';
comma = ",";
}
query << ';';
//query << "\nCOMMIT;";
conn.query_void(query.str());
const auto& itm = parData[z];
conn.query_void(query,
itm.path,
tiger_to_string(itm.hash),
itm.level,
itm.is_directory,
itm.is_symlink,
itm.size,
system_clock::from_time_t(itm.atime),
system_clock::from_time_t(itm.mtime),
itm.hash_valid,
itm.unreadable
);
}
conn.query_void("COMMIT;");
}

31
src/scan/dbbackend.hpp
View file

@ -21,40 +21,19 @@
#include <string>
#include <vector>
#include <cstdint>
#include <boost/utility/string_ref.hpp>
#include <ctime>
namespace dinlib {
struct SettingsDB;;
} //namespace dinlib
namespace din {
struct FileRecordData {
std::string path;
std::string hash;
std::time_t atime;
std::time_t mtime;
uint16_t level;
uint64_t size;
bool is_directory;
bool is_symlink;
bool unreadable;
bool hash_valid;
};
struct SetRecordDataFull {
std::string name;
uint32_t disk_number;
char type;
};
struct SetRecordData {
const boost::string_ref name;
const char type;
};
struct FileRecordData;
struct SetRecordData;
struct SetRecordDataFull;
struct TigerHash;
void write_to_db ( const dinlib::SettingsDB& parDB, const std::vector<FileRecordData>& parData, const SetRecordData& parSetData );
bool read_from_db ( FileRecordData& parItem, SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, std::string&& parHash );
bool read_from_db ( FileRecordData& parItem, SetRecordDataFull& parSet, const dinlib::SettingsDB& parDB, const TigerHash& parHash );
} //namespace din
#endif

128
src/scan/indexer.cpp
View file

@ -22,6 +22,7 @@
#include "dindexer-common/settings.hpp"
#include "filestats.hpp"
#include "mimetype.hpp"
#include "recorddata.hpp"
#include <algorithm>
#include <functional>
#include <vector>
@ -40,50 +41,18 @@
#if defined(INDEXER_VERBOSE)
# include <iostream>
#endif
#include <boost/utility/string_ref.hpp>
namespace din {
typedef TigerHash HashType;
struct FileEntry {
FileEntry ( const char* parPath, const fastf::FileStats& parSt ) :
path(parPath),
hash {},
access_time(parSt.atime),
modify_time(parSt.mtime),
//file_size(0),
level(static_cast<uint16_t>(parSt.level)),
is_dir(parSt.is_dir),
is_symlink(parSt.is_symlink),
unreadable(false)
{
}
FileEntry ( const FileEntry& ) = delete;
FileEntry ( FileEntry&& ) = default;
FileEntry& operator= ( const FileEntry& ) = delete;
FileEntry& operator= ( FileEntry&& ) = default;
bool operator< ( const FileEntry& parOther ) const;
bool operator== ( const FileEntry& ) const = delete;
std::string path;
std::string mime;
HashType hash;
std::time_t access_time;
std::time_t modify_time;
uint64_t file_size;
uint16_t level;
bool is_dir;
bool is_symlink;
bool unreadable;
};
using HashType = decltype(FileRecordData::hash);
namespace {
typedef std::vector<FileEntry>::iterator FileEntryIt;
typedef std::vector<FileRecordData>::iterator FileEntryIt;
void hash_dir (FileEntryIt parEntry, FileEntryIt parBegin, FileEntryIt parEnd, const PathName& parCurrDir, std::function<void(std::size_t)> parNextItemCallback, bool parIgnoreErrors, MimeType& parMime) {
assert(parEntry != parEnd);
assert(parEntry->is_dir);
FileEntry& curr_entry = *parEntry;
assert(parEntry->is_directory);
FileRecordData& curr_entry = *parEntry;
auto& curr_entry_it = parEntry;
//Build a blob with the hashes and filenames of every directory that
@ -106,10 +75,10 @@ namespace din {
#if defined(INDEXER_VERBOSE)
std::cout << "Making initial hash for " << parCurrDir << "...\n";
#endif
curr_entry.mime = parMime.analyze(it_entry->path);
curr_entry.mime_full = parMime.analyze(it_entry->path);
while (parEnd != it_entry and it_entry->level == curr_entry_it->level + 1 and parCurrDir == PathName(it_entry->path).pop_right()) {
PathName curr_subdir(it_entry->path);
if (it_entry->is_dir) {
if (it_entry->is_directory) {
hash_dir(it_entry, parBegin, parEnd, curr_subdir, parNextItemCallback, parIgnoreErrors, parMime);
std::string relpath = make_relative_path(parCurrDir, curr_subdir).path();
@ -128,11 +97,11 @@ namespace din {
}
tiger_data(dir_blob, curr_entry.hash);
curr_entry.file_size = 0;
curr_entry.size = 0;
#if defined(INDEXER_VERBOSE)
std::cout << "Got intermediate hash for dir " << parCurrDir <<
": " << tiger_to_string(curr_entry.hash) <<
' ' << curr_entry.mime << '\n';
' ' << curr_entry.mime_type << '\n';
#endif
}
@ -141,7 +110,7 @@ namespace din {
auto it_entry = curr_entry_it;
while (
it_entry != parEnd
and (it_entry->is_dir
and (it_entry->is_directory
or it_entry->level != curr_entry_it->level + 1
or PathName(it_entry->path).pop_right() != parCurrDir
)
@ -149,15 +118,19 @@ namespace din {
++it_entry;
}
while (it_entry != parEnd and not it_entry->is_dir and it_entry->level == curr_entry_it->level + 1 and PathName(it_entry->path).pop_right() == parCurrDir) {
assert(not it_entry->is_dir);
while (it_entry != parEnd and not it_entry->is_directory and it_entry->level == curr_entry_it->level + 1 and PathName(it_entry->path).pop_right() == parCurrDir) {
assert(not it_entry->is_directory);
#if defined(INDEXER_VERBOSE)
std::cout << "Hashing file " << it_entry->path << "...";
#endif
parNextItemCallback(it_entry - parBegin);
try {
tiger_file(it_entry->path, it_entry->hash, curr_entry_it->hash, it_entry->file_size);
it_entry->mime = parMime.analyze(it_entry->path);
tiger_file(it_entry->path, it_entry->hash, curr_entry_it->hash, it_entry->size);
it_entry->hash_valid = true;
it_entry->mime_full = parMime.analyze(it_entry->path);
auto mime_pair = split_mime(it_entry->mime_full);
it_entry->mime_type = mime_pair.first;
it_entry->mime_charset = mime_pair.second;
}
catch (const std::ios_base::failure& e) {
if (parIgnoreErrors) {
@ -170,8 +143,8 @@ namespace din {
}
#if defined(INDEXER_VERBOSE)
std::cout << ' ' << tiger_to_string(it_entry->hash) <<
' ' << it_entry->mime << '\n';
std::cout << ' ' << tiger_to_string(it_entry->hash) << ' ' <<
"Mime type: \"" << it_entry->mime_type << "\"\n";
#endif
++it_entry;
}
@ -180,16 +153,28 @@ namespace din {
#if defined(INDEXER_VERBOSE)
std::cout << "Final hash for dir " << parCurrDir << " is " << tiger_to_string(curr_entry_it->hash) << '\n';
#endif
curr_entry_it->hash_valid = true;
}
template <bool FileTrue=true>
struct IsFile {
bool operator() ( const FileEntry& parEntry ) const { return parEntry.is_dir xor FileTrue; }
bool operator() ( const FileRecordData& parEntry ) const { return parEntry.is_directory xor FileTrue; }
};
FileRecordData make_file_record_data (const char* parPath, const fastf::FileStats& parSt) {
return FileRecordData(
parPath,
parSt.atime,
parSt.mtime,
parSt.level,
parSt.is_dir,
parSt.is_symlink
);
}
} //unnamed namespace
struct Indexer::LocalData {
typedef std::vector<FileEntry> PathList;
typedef std::vector<FileRecordData> PathList;
dinlib::SettingsDB db_settings;
PathList paths;
@ -202,12 +187,13 @@ namespace din {
bool ignore_read_errors;
};
bool FileEntry::operator< (const FileEntry& parOther) const {
const FileEntry& o = parOther;
bool file_record_data_lt (const FileRecordData& parLeft, const FileRecordData& parRight) {
const FileRecordData& l = parLeft;
const FileRecordData& r = parRight;
return
(level < o.level)
or (level == o.level and is_dir and not o.is_dir)
or (level == o.level and is_dir == o.is_dir and path < o.path)
(l.level < r.level)
or (l.level == r.level and l.is_directory and not r.is_directory)
or (l.level == r.level and l.is_directory == r.is_directory and l.path < r.path)
//sort by directory - parent first, children later
//(level == o.level and is_dir and not o.is_dir)
@ -257,7 +243,7 @@ namespace din {
void Indexer::calculate_hash() {
PathName base_path(m_local_data->paths.front().path);
std::sort(m_local_data->paths.begin(), m_local_data->paths.end());
std::sort(m_local_data->paths.begin(), m_local_data->paths.end(), &file_record_data_lt);
MimeType mime;
#if defined(INDEXER_VERBOSE)
@ -266,7 +252,7 @@ namespace din {
itm.hash.part_b = 1;
itm.hash.part_c = 1;
if (itm.is_dir)
if (itm.is_directory)
std::cout << "(D) ";
else
std::cout << "(F) ";
@ -317,41 +303,25 @@ namespace din {
#endif
if (not parForce) {
std::string first_hash(tiger_to_string(m_local_data->paths.front().hash, true));
const auto& first_hash = m_local_data->paths.front().hash;
FileRecordData itm;
SetRecordDataFull set;
const bool already_in_db = read_from_db(itm, set, m_local_data->db_settings, std::move(first_hash));
const bool already_in_db = read_from_db(itm, set, m_local_data->db_settings, first_hash);
if (already_in_db) {
return false;
}
}
PathName base_path(m_local_data->paths.front().path);
std::vector<FileRecordData> data;
data.reserve(m_local_data->paths.size());
for (const auto& itm : m_local_data->paths) {
data.push_back(FileRecordData {
make_relative_path(base_path, PathName(itm.path)).path(),
tiger_to_string(itm.hash),
itm.access_time,
itm.modify_time,
itm.level,
itm.file_size,
itm.is_dir,
itm.is_symlink,
itm.unreadable,
not itm.unreadable
});
}
SetRecordData set_data {parSetName, parType};
write_to_db(m_local_data->db_settings, data, set_data);
write_to_db(m_local_data->db_settings, m_local_data->paths, set_data);
return true;
}
bool Indexer::add_path (const char* parPath, const fastf::FileStats& parStats) {
m_local_data->paths.push_back(
FileEntry(parPath, parStats));
make_file_record_data(parPath, parStats));
if (not parStats.is_dir) {
++m_local_data->file_count;
}
@ -364,14 +334,14 @@ namespace din {
std::cout << "---------------- FILE LIST ----------------\n";
for (const auto& cur_itm : m_local_data->paths) {
if (not cur_itm.is_dir) {
if (not cur_itm.is_directory) {
PathName cur_path(cur_itm.path);
std::cout << make_relative_path(base_path, cur_path).path() << '\n';
}
}
std::cout << "---------------- DIRECTORY LIST ----------------\n";
for (const auto& cur_itm : m_local_data->paths) {
if (cur_itm.is_dir) {
if (cur_itm.is_directory) {
PathName cur_path(cur_itm.path);
std::cout << make_relative_path(base_path, cur_path).path() << '\n';
}

4
src/scan/main.cpp
View file

@ -133,6 +133,10 @@ namespace {
parShowProgress = false;
#endif
if (not parShowProgress) {
//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1
//terminate called after throwing an instance of 'std::ios_base::failure'
//what(): basic_filebuf::xsgetn error reading the file
//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato
parIndexer.calculate_hash();
}
#if defined(WITH_PROGRESS_FEEDBACK)

86
src/scan/recorddata.hpp Normal file
View file

@ -0,0 +1,86 @@
/* Copyright 2015, Michele Santullo
* This file is part of "dindexer".
*
* "dindexer" is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* "dindexer" is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with "dindexer". If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef id3CD7F105AC314540A864487E981E5A7E
#define id3CD7F105AC314540A864487E981E5A7E
#include "tiger.hpp"
#include <string>
#include <boost/utility/string_ref.hpp>
#include <cstdint>
#include <ctime>
#include <boost/flyweight.hpp>
#include <boost/flyweight/no_locking.hpp>
namespace din {
struct FileRecordData {
struct MimeStringTagStruct { };
typedef boost::flyweights::tag<MimeStringTagStruct> MimeStringTag;
typedef boost::flyweight<std::string, boost::flyweights::no_locking, MimeStringTag> mime_string;
FileRecordData ( void ) = default;
FileRecordData ( const char* parPath, std::time_t parATime, std::time_t parMTime, uint64_t parLevel, bool parIsDir, bool parIsSymLink ) :
hash {},
path(parPath),
mime_full(),
atime(parATime),
mtime(parMTime),
mime_type(),
mime_charset(),
size(0),
level(parLevel),
is_directory(parIsDir),
is_symlink(parIsSymLink),
unreadable(false),
hash_valid(false)
{
}
FileRecordData ( const FileRecordData& ) = delete;
FileRecordData ( FileRecordData&& ) = default;
FileRecordData& operator= ( const FileRecordData& ) = delete;
FileRecordData& operator= ( FileRecordData&& ) = default;
bool operator== ( const FileRecordData& ) const = delete;
TigerHash hash;
std::string path;
mime_string mime_full;
std::time_t atime;
std::time_t mtime;
boost::string_ref mime_type;
boost::string_ref mime_charset;
uint64_t size;
uint16_t level;
bool is_directory;
bool is_symlink;
bool unreadable;
bool hash_valid;
};
struct SetRecordDataFull {
std::string name;
uint32_t disk_number;
char type;
};
struct SetRecordData {
const boost::string_ref name;
const char type;
};
} //namespace din
#endif