Create DirTree class and start moving code around.

cscope_gen.sh

@@ -32,4 +32,4 @@ set -f
 find . \( $excl_paths -o $incl_extensions \) -a -type f $excl_files > cscope.files
 set +f
 
-cscope -b -q
+exec cscope -b -q

include/dindexer-machinery/scantask/base.hpp

@@ -0,0 +1,68 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef idCB253C1A5AFA46A18B8878ED4072CD96
+#define idCB253C1A5AFA46A18B8878ED4072CD96
+
+#include <ciso646>
+
+namespace mchlib {
+	namespace scantask {
+		template <typename T>
+		class Base {
+		protected:
+			Base ( void );
+			virtual ~Base ( void ) noexcept = default;
+
+		public:
+			T& get_or_create ( void );
+			void clear_data ( void );
+
+		private:
+			virtual void on_data_destroy ( T& parData ) = 0;
+			virtual void on_data_create ( T& parData ) = 0;
+
+			T m_data;
+			bool m_data_created;
+		};
+
+		template <typename T>
+		Base<T>::Base() :
+			m_data_created(false)
+		{
+		}
+
+		template <typename T>
+		T& Base<T>::get_or_create() {
+			if (not m_data_created) {
+				m_data_created = true;
+				this->on_data_create(m_data);
+			}
+			return m_data;
+		}
+
+		template <typename T>
+		void Base<T>::clear_data() {
+			if (m_data_created) {
+				m_data_created = false;
+				this->on_data_destroy(m_data);
+			}
+		}
+	} //namespace scantask
+} //namespace mchlib
+
+#endif

include/dindexer-machinery/scantask/dirtree.hpp

@@ -0,0 +1,45 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef id0AA31B2E7D6244A08435CF9080E34AAE
+#define id0AA31B2E7D6244A08435CF9080E34AAE
+
+#include "dindexer-machinery/scantask/base.hpp"
+#include <string>
+#include <vector>
+
+namespace mchlib {
+	struct FileRecordData;
+
+	namespace scantask {
+		class DirTree : public Base<std::vector<FileRecordData>> {
+		public:
+			typedef std::vector<FileRecordData> PathList;
+
+			explicit DirTree ( std::string parRoot );
+			virtual ~DirTree ( void ) noexcept = default;
+
+		private:
+			virtual void on_data_destroy ( PathList& parData );
+			virtual void on_data_create ( PathList& parData );
+
+			std::string m_root;
+		};
+	} //namespace scantask
+} //namespace mchlib
+
+#endif

src/machinery/CMakeLists.txt

@@ -4,7 +4,6 @@ include(WithMediaAutodetect)
 find_package(Magic REQUIRED)
 
 add_library(${PROJECT_NAME} SHARED
-	indexer.cpp
 	pathname.cpp
 	tiger.c
 	tiger.cpp
@@ -16,6 +15,7 @@ add_library(${PROJECT_NAME} SHARED
 	guess_content_type.cpp
 	set_listing.cpp
 	globbing.cpp
+	scantask/dirtree.cpp
 )
 
 #target_include_directories(${PROJECT_NAME}
@@ -29,6 +29,10 @@ target_link_libraries(${PROJECT_NAME}
 	PRIVATE ${MAGIC_LIBRARIES}
 )
 
+target_include_directories(${PROJECT_NAME}
+	PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
 if (DINDEXER_WITH_MEDIA_AUTODETECT)
 	target_include_directories(${PROJECT_NAME} SYSTEM
 		PRIVATE ${BLKID_INCLUDE_DIRS}

src/machinery/filesearcher.cpp

@@ -15,7 +15,7 @@
  * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "dindexer-machinery/filesearcher.hpp"
+#include "filesearcher.hpp"
 
 #if !defined(_XOPEN_SOURCE)
 #define _XOPEN_SOURCE 500

src/machinery/scantask/dirtree.cpp

@@ -0,0 +1,81 @@
+/* Copyright 2015, 2016, Michele Santullo
+ * This file is part of "dindexer".
+ *
+ * "dindexer" is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * "dindexer" is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with "dindexer".  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dindexer-machinery/scantask/dirtree.hpp"
+#include "dindexer-machinery/recorddata.hpp"
+#include "dindexer-machinery/set_listing.hpp"
+#include "filesearcher.hpp"
+#include <utility>
+#include <cassert>
+#include <ciso646>
+#include <functional>
+
+namespace mchlib {
+	namespace {
+		FileRecordData make_file_record_data (const char* parPath, const fastf::FileStats& parSt) {
+			return FileRecordData(
+				parPath,
+				parSt.atime,
+				parSt.mtime,
+				parSt.level,
+				parSt.is_dir,
+				parSt.is_symlink
+			);
+		}
+
+		bool add_path (scantask::DirTree::PathList& parOut, const char* parPath, const fastf::FileStats& parStats) {
+			auto it_before = SetListing::lower_bound(
+				parOut,
+				parPath,
+				parStats.level,
+				parStats.is_dir
+			);
+
+			parOut.insert(
+				it_before,
+				make_file_record_data(parPath, parStats)
+			);
+			return true;
+		}
+	}
+
+	namespace scantask {
+		DirTree::DirTree (std::string parRoot) :
+			m_root(std::move(parRoot))
+		{
+			assert(not m_root.empty());
+		}
+
+		void DirTree::on_data_destroy (PathList& parData) {
+			parData.clear();
+		}
+
+		void DirTree::on_data_create (PathList& parData) {
+			using std::placeholders::_1;
+			using std::placeholders::_2;
+
+			assert(parData.empty());
+
+			fastf::FileSearcher searcher(m_root);
+			fastf::FileSearcher::ConstCharVecType ext, ignore;
+
+			searcher.SetFollowSymlinks(true);
+			searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&add_path, std::ref(parData), _1, _2)));
+			searcher.Search(ext, ignore);
+		}
+	} //namespace scantask
+} //namespace mchlib

src/scan/main.cpp

@@ -21,14 +21,13 @@
 
 #include "dindexer-machinery/recorddata.hpp"
 #include "dindexerConfig.h"
-#include "dindexer-machinery/filesearcher.hpp"
-#include "dindexer-machinery/indexer.hpp"
 #include "dindexer-machinery/machinery_info.hpp"
 #include "dindexer-common/common_info.hpp"
 #include "dindexer-common/settings.hpp"
-#include "dindexer-machinery/guess_content_type.hpp"
+//#include "dindexer-machinery/guess_content_type.hpp"
 #include "commandline.hpp"
 #include "dbbackend.hpp"
+#include "dindexer-machinery/scantask/dirtree.hpp"
 #include <iostream>
 #include <iomanip>
 #include <ciso646>
@@ -41,10 +40,10 @@
 #	include <condition_variable>
 #endif
 
-namespace {
+//namespace {
-	void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
+//	void run_hash_calculation ( mchlib::Indexer& parIndexer, bool parShowProgress );
-	bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
+//	bool add_to_db ( const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContent, const dinlib::SettingsDB& parDBSettings, bool parForce=false );
-} //unnamed namespace
+//} //unnamed namespace
 
 int main (int parArgc, char* parArgv[]) {
 	using std::placeholders::_1;
@@ -61,7 +60,6 @@ int main (int parArgc, char* parArgv[]) {
 		std::cerr << err.what() << "\nUse --help for help" << std::endl;
 		return 2;
 	}
-	const std::string search_path(vm["search-path"].as<std::string>());
 #if defined(WITH_PROGRESS_FEEDBACK)
 	const bool verbose = (0 == vm.count("quiet"));
 #else
@@ -77,48 +75,49 @@ int main (int parArgc, char* parArgv[]) {
 		}
 	}
 
+	const std::string search_path(vm["search-path"].as<std::string>());
+	mchlib::scantask::DirTree scan_dirtree(search_path);
+
 #if defined(WITH_MEDIA_AUTODETECT)
-	char set_type;
+	//char set_type;
-	if (0 == vm.count("type")) {
+	//if (0 == vm.count("type")) {
-		std::cout << "Analyzing disc... ";
+	//	std::cout << "Analyzing disc... ";
-		try {
+	//	try {
-			const auto guessed_type = mchlib::guess_media_type(std::string(search_path));
+	//		const auto guessed_type = mchlib::guess_media_type(std::string(search_path));
-			set_type = guessed_type;
+	//		set_type = guessed_type;
-			std::cout << "Setting type to " << set_type << " ("
+	//		std::cout << "Setting type to " << set_type << " ("
-				<< dinlib::media_type_to_str(guessed_type) << ")\n";
+	//			<< dinlib::media_type_to_str(guessed_type) << ")\n";
-		}
+	//	}
-		catch (const std::runtime_error& e) {
+	//	catch (const std::runtime_error& e) {
-			std::cout << '\n';
+	//		std::cout << '\n';
-			std::cerr << e.what();
+	//		std::cerr << e.what();
-			return 1;
+	//		return 1;
-		}
+	//	}
-	}
+	//}
-	else {
+	//else {
-		set_type = vm["type"].as<char>();
+	//	set_type = vm["type"].as<char>();
-	}
+	//}
+	//std::unique_ptr<mchlib::scantask::MediaAutodetect> media_autodetector(
+		//new mchlib::scantask::MediaAutodetect(vm["type"].as<char>())
+	//);
 #else
 	const char set_type = vm["type"].as<char>();
 #endif
 
 	std::cout << "constructing...\n";
 
-	mchlib::Indexer indexer;
+	//indexer.ignore_read_errors(vm.count("ignore-errors") > 0);
-	indexer.ignore_read_errors(vm.count("ignore-errors") > 0);
+	//if (verbose) {
-	fastf::FileSearcher searcher(search_path);
+	//	std::cout << "Fetching items list...\n";
-	fastf::FileSearcher::ConstCharVecType ext, ignore;
+	//}
-	searcher.SetFollowSymlinks(true);
-	searcher.SetCallback(fastf::FileSearcher::CallbackType(std::bind(&mchlib::Indexer::add_path, &indexer, _1, _2)));
-	searcher.Search(ext, ignore);
-	if (verbose) {
-		std::cout << "Fetching items list...\n";
-	}
 
-	if (indexer.empty()) {
+	//if (indexer.empty()) {
-		std::cerr << "Nothing found at the given location, quitting\n";
+	//	std::cerr << "Nothing found at the given location, quitting\n";
-		return 1;
+	//	return 1;
-	}
+	//}
-	else {
+	//else {
-		run_hash_calculation(indexer, verbose);
+	{
+		//run_hash_calculation(indexer, verbose);
 
 		//TODO: guess_content_type() relies on FileRecordData::path being set to
 		//the relative path already. Unfortunately at this point it just got
@@ -128,95 +127,95 @@ int main (int parArgc, char* parArgv[]) {
 		//paths are populated at the end of calculate_hash(), so until I come up
 		//with a better system I'm just moving content detection to after hash
 		//calculation.
-		const auto set_type_casted = dinlib::char_to_media_type(set_type);
+		//const auto set_type_casted = dinlib::char_to_media_type(set_type);
-		const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data());
+		//const mchlib::ContentTypes content = mchlib::guess_content_type(set_type_casted, indexer.record_data());
-		const char content_type = mchlib::content_type_to_char(content);
+		//const char content_type = mchlib::content_type_to_char(content);
 
 		if (verbose) {
 			std::cout << "Writing to database...\n";
 		}
-		if (not add_to_db(indexer.record_data(), vm["setname"].as<std::string>(), set_type, content_type, settings.db)) {
+		//if (not add_to_db(indexer.record_data(), vm["setname"].as<std::string>(), set_type, content_type, settings.db)) {
-			std::cerr << "Not written to DB, likely because a set with the same hash already exists\n";
+		//	std::cerr << "Not written to DB, likely because a set with the same hash already exists\n";
-		}
+		//}
 	}
 	return 0;
 }
 
-namespace {
+//namespace {
-	void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
+//	void run_hash_calculation (mchlib::Indexer& parIndexer, bool parShowProgress) {
-		if (parIndexer.empty()) {
+//		if (parIndexer.empty()) {
-			return;
+//			return;
-		}
+//		}
-
+//
-#if !defined(WITH_PROGRESS_FEEDBACK)
+//#if !defined(WITH_PROGRESS_FEEDBACK)
-		parShowProgress = false;
+//		parShowProgress = false;
-#endif
+//#endif
-		if (not parShowProgress) {
+//		if (not parShowProgress) {
-//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1
+////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.md1... 512c201321ed01cc2a82c9f80bfeaaa673bc8eb3cea4e5c1
-//terminate called after throwing an instance of 'std::ios_base::failure'
+////terminate called after throwing an instance of 'std::ios_base::failure'
-//what():  basic_filebuf::xsgetn error reading the file
+////what():  basic_filebuf::xsgetn error reading the file
-//Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato
+////Hashing file /mnt/cdrom/Sacred 2/Fallen Angel/UK/Sacred.2.Fallen.Angel-ArenaBG/DISC2/S2DISC2.mdf...Annullato
-			parIndexer.calculate_hash();
+//			parIndexer.calculate_hash();
-		}
+//		}
-#if defined(WITH_PROGRESS_FEEDBACK)
+//#if defined(WITH_PROGRESS_FEEDBACK)
-		else {
+//		else {
-			typedef std::ostream_iterator<char> cout_iterator;
+//			typedef std::ostream_iterator<char> cout_iterator;
-
+//
-			std::cout << "Processing";
+//			std::cout << "Processing";
-			std::cout.flush();
+//			std::cout.flush();
-			const auto total_items = parIndexer.total_items();
+//			const auto total_items = parIndexer.total_items();
-			std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer);
+//			std::thread hash_thread(&mchlib::Indexer::calculate_hash, &parIndexer);
-			std::mutex progress_print;
+//			std::mutex progress_print;
-			std::size_t clear_size = 0;
+//			std::size_t clear_size = 0;
-			const auto digit_count = static_cast<std::size_t>(std::log10(static_cast<double>(total_items))) + 1;
+//			const auto digit_count = static_cast<std::size_t>(std::log10(static_cast<double>(total_items))) + 1;
-			do {
+//			do {
-				//TODO: fix this steaming pile of crap
+//				//TODO: fix this steaming pile of crap
-				//std::unique_lock<std::mutex> lk(progress_print);
+//				//std::unique_lock<std::mutex> lk(progress_print);
-				//parIndexer.step_notify().wait(lk);
+//				//parIndexer.step_notify().wait(lk);
-				std::cout << '\r';
+//				std::cout << '\r';
-				std::fill_n(cout_iterator(std::cout), clear_size, ' ');
+//				std::fill_n(cout_iterator(std::cout), clear_size, ' ');
-				std::cout << '\r';
+//				std::cout << '\r';
-				{
+//				{
-					std::ostringstream oss;
+//					std::ostringstream oss;
-					const auto item_index = std::min(total_items - 1, parIndexer.processed_items());
+//					const auto item_index = std::min(total_items - 1, parIndexer.processed_items());
-					oss << "Processing file "
+//					oss << "Processing file "
-						<< std::setw(digit_count) << std::setfill(' ') << (item_index + 1)
+//						<< std::setw(digit_count) << std::setfill(' ') << (item_index + 1)
-						<< " of " << total_items << " \"" << parIndexer.current_item() << '"';
+//						<< " of " << total_items << " \"" << parIndexer.current_item() << '"';
-					const auto msg(oss.str());
+//					const auto msg(oss.str());
-					clear_size = msg.size();
+//					clear_size = msg.size();
-					std::cout << msg;
+//					std::cout << msg;
-					std::cout.flush();
+//					std::cout.flush();
-				}
+//				}
-			} while (false); //parIndexer.processed_items() != total_items);
+//			} while (false); //parIndexer.processed_items() != total_items);
-
+//
-			hash_thread.join();
+//			hash_thread.join();
-			if (parIndexer.processed_items() > 0) {
+//			if (parIndexer.processed_items() > 0) {
-				std::cout << '\n';
+//				std::cout << '\n';
-			}
+//			}
-		}
+//		}
-#endif
+//#endif
-	}
+//	}
-
+//
-	bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) {
+//	bool add_to_db (const std::vector<mchlib::FileRecordData>& parData, const std::string& parSetName, char parType, char parContentType, const dinlib::SettingsDB& parDBSettings, bool parForce) {
-		using mchlib::FileRecordData;
+//		using mchlib::FileRecordData;
-		using mchlib::SetRecordDataFull;
+//		using mchlib::SetRecordDataFull;
-		using mchlib::SetRecordData;
+//		using mchlib::SetRecordData;
-
+//
-		if (not parForce) {
+//		if (not parForce) {
-			const auto& first_hash = parData.front().hash;
+//			const auto& first_hash = parData.front().hash;
-			FileRecordData itm;
+//			FileRecordData itm;
-			SetRecordDataFull set;
+//			SetRecordDataFull set;
-			const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash);
+//			const bool already_in_db = din::read_from_db(itm, set, parDBSettings, first_hash);
-			if (already_in_db) {
+//			if (already_in_db) {
-				return false;
+//				return false;
-			}
+//			}
-		}
+//		}
-
+//
-		SetRecordData set_data {parSetName, parType, parContentType };
+//		SetRecordData set_data {parSetName, parType, parContentType };
-		const auto app_signature = dinlib::dindexer_signature();
+//		const auto app_signature = dinlib::dindexer_signature();
-		const auto lib_signature = mchlib::lib_signature();
+//		const auto lib_signature = mchlib::lib_signature();
-		const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
+//		const std::string signature = std::string(app_signature.data(), app_signature.size()) + "/" + std::string(lib_signature.data(), lib_signature.size());
-		din::write_to_db(parDBSettings, parData, set_data, signature);
+//		din::write_to_db(parDBSettings, parData, set_data, signature);
-		return true;
+//		return true;
-	}
+//	}
-} //unnamed namespace
+//} //unnamed namespace