King_DuckZ/duckscraper
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Keep FromBlock data in tree form.

Browse source
This commit is contained in:
King_DuckZ 2018-01-16 19:50:35 +00:00
parent 26b912d66c
commit 2fd4daf52c
1 changed files with 54 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
src/scraplang/apply.cpp
View file

@ -27,48 +27,66 @@
#endif
#include <map>
#include <boost/variant/apply_visitor.hpp>
#include <string_view>
namespace duck { namespace sl {
#if defined(APPLY_VERBOSE)
#endif
namespace {
struct EntryNode;
struct ApplyEntry {
const SourceInfo* apply_to;
const std::vector<StructItem>* content;
const std::string* mustache_name;
};
typedef std::map<std::string, const XPathElement*> XPathMap;
struct MustacheEntry {
std::string text;
mstch::map context;
};
using EntryNodeList = std::vector<std::pair<
const SourceInfo*,
std::vector<EntryNode>
>>;
using MustacheEntryMap = std::map<std::string, MustacheEntry>;
struct EntryNode {
explicit EntryNode (const std::string& parName) :
name(parName)
{
}
EntryNode (EntryNode&&) = default;
EntryNode& operator= (EntryNode&&) = default;
std::string_view name;
std::vector<EntryNode> structs;
std::vector<const XPathElement*> xpaths;
};
class StructItemExtractor : public boost::static_visitor<> {
public:
StructItemExtractor (std::string&& parPrefix, XPathMap& parMap) :
m_prefix(std::move(parPrefix)),
m_map(parMap)
explicit StructItemExtractor (EntryNode& parRoot) :
m_root(parRoot)
{
if (not m_prefix.empty()) {
m_prefix += ".";
}
}
void operator() (const XPathElement& parVal) {
m_map[m_prefix + parVal.name] = &parVal;
m_root.xpaths.push_back(&parVal);
}
void operator() (const StructBlock& parVal) {
StructItemExtractor visitor(
m_prefix + parVal.name,
m_map
);
m_root.structs.emplace_back(parVal.name);
StructItemExtractor visitor(m_root.structs.back());
for (auto& itm : parVal.xpaths) {
boost::apply_visitor(visitor, itm);
}
}
private:
std::string m_prefix;
XPathMap& m_map;
EntryNode& m_root;
};
class DictBuilder : public boost::static_visitor<> {
@ -89,8 +107,15 @@ namespace duck { namespace sl {
#if defined(APPLY_VERBOSE)
std::cout << parVal << '\n';
#endif
m_global_entries.emplace_back(std::make_pair(
&parVal.source,
std::vector<EntryNode>()
));
auto& curr_list = m_global_entries.back().second;
for (auto& itm : parVal.xpaths) {
StructItemExtractor extractor("", m_global_entries);
curr_list.emplace_back("");
StructItemExtractor extractor(curr_list.back());
boost::apply_visitor(extractor, itm);
}
}
@ -122,17 +147,13 @@ namespace duck { namespace sl {
m_current_mustache->text = parVal.content;
}
const XPathMap& global_entries() const { return m_global_entries; }
const EntryNodeList& global_entries() const { return m_global_entries; }
const MustacheEntryMap& mustache_entries() const { return m_mustaches; }
private:
struct MustacheEntry {
std::string text;
mstch::map context;
};
XPathMap m_global_entries;
EntryNodeList m_global_entries;
std::vector<ApplyEntry> m_apply_entries;
std::map<std::string, MustacheEntry> m_mustaches;
MustacheEntryMap m_mustaches;
const std::string* m_current_mustache_name;
MustacheEntry* m_current_mustache;
};
@ -143,16 +164,20 @@ namespace duck { namespace sl {
HtmlPoolBaseSP html_pool
) {
DictBuilder dict_builder(html_pool);
boost::apply_visitor(dict_builder, node);
std::vector<std::string> retval;
const EntryNodeList& global_entries = dict_builder.global_entries();
const MustacheEntryMap& mustaches = dict_builder.mustache_entries();
retval.reserve(mustaches.size());
std::cout << "-------------- visiting done ----------------\n";
for (auto& itm : dict_builder.global_entries()) {
std::cout << "item: \"" << itm.first << "\", \"" <<
itm.second->xpath << "\"\n";
}
//for (auto& itm : dict_builder.global_entries()) {
// std::cout << "item: \"" << itm.first << "\", \"" <<
// itm.second->xpath << "\"\n";
//}
return std::vector<std::string>();
return retval;
}
}} //namespace duck::sl