King_DuckZ/dindexer
1
0
Fork 0
mirror of https://github.com/KingDuckZ/dindexer.git synced 2024-11-29 01:33:46 +00:00
Code Issues Releases Wiki Activity

Limit insertion to 100 records per query, but this is still suboptimal.

Browse source
This commit is contained in:
King_DuckZ 2015-11-11 12:35:52 +00:00
parent 4236b2ece8
commit 444d2a2fa3
1 changed files with 23 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
src/dbbackend.cpp
View file

@ -23,6 +23,7 @@
namespace din {
namespace {
const std::size_t g_batch_size = 100;
} //unnamed namespace
void write_to_db (const std::vector<FileRecordData>& parData) {
@ -30,26 +31,31 @@ namespace din {
return;
}
std::ostringstream query;
query << "BEGIN;\n";
query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES ";
pq::Connection conn("michele", "password", "dindexer", "100.200.100.200", 5432);
conn.connect();
const char* comma = "";
for (const auto& itm : parData) {
query << comma;
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ','
<< 10 << ',' << (itm.is_directory ? "true" : "false") << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')'
;
comma = ",";
}
query << ';';
query << "\nCOMMIT;";
conn.query_void("BEGIN;");
//TODO: use COPY instead of INSERT INTO
for (std::size_t z = 0; z < parData.size(); z += g_batch_size) {
std::ostringstream query;
query << "INSERT INTO \"Files\" (path, hash, level, group_id, is_directory, is_symlink, size) VALUES ";
conn.query_void(query.str());
const char* comma = "";
for (auto i = z; i < std::min(z + g_batch_size, parData.size()); ++i) {
const auto& itm = parData[i];
query << comma;
query << '(' << conn.escape_literal(itm.path) << ",'" << itm.hash << "',"
<< itm.level << ','
<< 10 << ',' << (itm.is_directory ? "true" : "false") << ','
<< (itm.is_symlink ? "true" : "false") << ',' << itm.size << ')'
;
comma = ",";
}
query << ';';
//query << "\nCOMMIT;";
conn.query_void(query.str());
}
conn.query_void("COMMIT;");
}
} //namespace din