King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2025-07-03 14:14:11 +00:00
Code Issues Releases Wiki Activity

Limit insertion to 100 records per query, but this is still suboptimal.

Browse source
This commit is contained in:
King_DuckZ 2015-11-11 12:35:52 +00:00
parent 4236b2ece8
commit 444d2a2fa3
1 changed files with 23 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/dbbackend.cpp
View file

@ -23,6 +23,7 @@
namespace din { namespace din {
namespace { namespace {
const std::size_t g_batch_size = 100;
} //unnamed namespace } //unnamed namespace
void write_to_db (const std::vector<FileRecordData>& parData) { void write_to_db (const std::vector<FileRecordData>& parData) {
@ -30,26 +31,31 @@ namespace din {
return; return;
} }
std::ostringstream query;
query << "BEGIN;\n";
query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES ";
pq::Connection conn("michele", "password", "dindexer", "100.200.100.200", 5432); pq::Connection conn("michele", "password", "dindexer", "100.200.100.200", 5432);
conn.connect(); conn.connect();
const char* comma = ""; conn.query_void("BEGIN;");
for (const auto& itm : parData) { //TODO: use COPY instead of INSERT INTO
query << comma; for (std::size_t z = 0; z < parData.size(); z += g_batch_size) {
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "'," std::ostringstream query;
<< itm.level << ',' query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES ";
<< 10 << ',' << (itm.is_directory ? "true" : "false") << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')'
;
comma = ",";
}
query << ';';
query << "\nCOMMIT;";
conn.query_void(query.str()); const char* comma = "";
for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) {
const auto& itm = parData[i];
query << comma;
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ','
<< 10 << ',' << (itm.is_directory ? "true" : "false") << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')'
;
comma = ",";
}
query << ';';
//query << "\nCOMMIT;";
conn.query_void(query.str());
}
conn.query_void("COMMIT;");
} }
} //namespace din } //namespace din