Boost库搜索引擎¶

// 结果基本内容结构
struct ResultData
{
    std::string title; // 结构标题
    std::string body; // 结果内容或描述
    std::string url; // 网址
};

- 根路径
    - data
        - source
            - html // 所有的HTML文件位于这里
        - raw // 结构化数据存储到这里
    // 头文件和源文件文件与data目录平级

class DataParse
{
private:

};

// HTML文件后缀
const std::string g_html_extension = ".html";

// 获取HTML文件路径函数
bool getHtmlSourceFiles()
{
    // 如果路径不存在，直接返回false
    if (!fs::exists(g_datasource_path))
    {
        ls::LOG(ls::LogLevel::DEBUG) << "不存在指定路径";
        return false;
    }

    for (const auto &entry : fs::recursive_directory_iterator(g_datasource_path))
    {
        // 1. 不是普通文件就是目录，继续遍历
        if (!fs::is_regular_file(entry))
            continue;

        // 2. 是普通文件，但是不是HTML文件，继续遍历
        if (entry.path().extension() != g_html_extension)
            continue;

        // 3. 是普通文件并且是HTML文件，插入结果
        sources_.push_back(std::move(entry.path()));
    }

    // 空结果返回false
    return !sources_.empty();
}

// 读取HTML文件内容
bool readInfoFromHtml()
{
    // 不存在路径返回false
    if (sources_.empty())
    {
        ls::LOG(ls::LogLevel::WARNING) << "文件路径不存在";
        return false;
    }

    // 每一个文件都执行四步
    for (const auto &path : sources_)
    {
        std::string out; // 存储HTML文件内容
        struct pd::ResultData rd;
        // 1. 读取文件
        if (!readHtmlFile(path, out))
        {
            ls::LOG(ls::LogLevel::WARNING) << "打开文件：" << path << "失败";
            // 读取当前文件失败时继续读取后面的文件
            continue;
        }

        // 2. 获取标题
        if (!getTitleFromHtml(out, &rd.title))
        {
            ls::LOG(ls::LogLevel::WARNING) << "获取文件标题失败";
            continue;
        }

        // 3. 读取文件内容
        if (!getContentFromHtml(out, &rd.body))
        {
            ls::LOG(ls::LogLevel::WARNING) << "获取文件内容失败";
            continue;
        }

        // 4. 构建URL
        if (!constructHtmlUrl(path, &rd.url))
        {
            ls::LOG(ls::LogLevel::WARNING) << "构建文件URL失败";
            continue;
        }
    }

    return true;
}

class DataParse
{
public:
    // ...

    // 从HTML文件中读取信息
    bool readInfoFromHtml()
    {
        // 不存在路径返回false
        if (sources_.empty())
        {
            ls::LOG(ls::LogLevel::WARNING) << "文件路径不存在";
            return false;
        }

        for (const auto &path : sources)
        {
            // ...

            results_.push_back(std::move(rd));
        }

        return true;
    }

private:
    // ...
    std::vector<pd::ResultData> results_;
};

// 读取HTML文件
bool readHtmlFile(const fs::path &p, std::string &out)
{
    if (p.empty())
        return false;

    std::fstream f(p);

    if (!f.is_open())
        return false;

    // 读取文件
    std::string buffer;
    while (getline(f, buffer))
        out += buffer;

    f.close();

    return true;
}

// 读取标题
// &为输入型参数，*为输出型参数
bool getTitleFromHtml(std::string &in, std::string *title)
{
    if(in.empty())
        return false;

    // 找到开始标签<title>
    auto start = in.find("<title>");
    if(start == std::string::npos)
        return false;

    // 找到终止标签</title>
    auto end = in.find("</title>");
    if (end == std::string::npos || start > end)
        return false;

    // 截取出其中的内容，左闭右开
    *title = in.substr(start + std::string("<title>").size(), end - (start + std::string("<title>").size()));

    return true;
}

// 内容状态
enum ContentStatus
{
    Label, // 标签状态
    OrdinaryContent // 普通文本状态
};

// 读取HTML文件内容
bool getContentFromHtml(std::string &out, std::string *body)
{
    // 默认状态为标签
    ContentStatus cs = ContentStatus::Label;

    // 注意，因为文档没有中文，直接用char没有问题
    for (char ch : out)
    {
        switch (cs)
        {
        // 读取到右尖括号且状态为标签说明接下来为文本内容
        case ContentStatus::Label:
            if(ch == '>')
                cs = ContentStatus::OrdinaryContent; // 切换状态
            break;
        case ContentStatus::OrdinaryContent:
            // 去除\n
            if (ch == '<')
                cs = ContentStatus::Label; // 切换状态
            else
            {
                if (ch == '\n')
                    *body += ' ';
                else 
                    *body += ch;
            }
            break;
        default:
            break;
        }
    }

    return true;
}

// 构建URL
bool constructHtmlUrl(const fs::path& p, std::string *url)
{
    // 在本地路径中找到"/文件"
    std::string t_path = p.string();
    auto pos = t_path.rfind("/");

    if(pos == std::string::npos)
        return false;

    // 从/开始截取一直到结尾
    std::string source_path = t_path.substr(pos);
    *url = g_url_to_concat + source_path;

    return true;
}

// 将结构体字段写入文本文件中
bool writeToRawFile()
{
    // 以二进制形式打开文件
    std::fstream f(g_rawfile_path);

    if (!f.is_open())
    {
        ls::LOG(ls::LogLevel::WARNING) << "文本文件不存在";
        return false;
    }

    // 写入结构化数据
    for (auto &rd : results_)
    {
        std::string temp;
        temp += rd.title;
        temp += g_rd_sep;
        temp += rd.body;
        temp += g_rd_sep;
        temp += rd.url;
        temp += g_html_sep;

        f.write(temp.c_str(), temp.size());
    }

    f.close();

    return true;
}

struct SelectedDocInfo
{
    struct pd::ResultData rd;
    uint64_t id;
};

// 倒排索引时当前关键字的信息
struct BackwardIndexElement
{
    uint64_t id; // 文档ID
    std::string word; // 关键字
    int weight; // 权重信息
};

// 构建索引
bool buildIndex()
{

}

class SearchIndex
{
public:
    // 获取正排索引结果
    SelectedDocInfo *getForwardIndexDocInfo()
    {
    }

    // 获取倒排索引结果
    std::vector<BackwardIndexElement> *getBackwardIndexElement()
    {
    }

    // 构建索引
    bool buildIndex()
    {

    }

private:
    std::vector<SelectedDocInfo> forward_index_; // 正排索引结果
    std::unordered_map<std::string, std::vector<BackwardIndexElement>> backward_index_; // 倒排索引结果
};

// 构建索引
bool buildIndex()
{
    // 以二进制方式读取文本文件中的内容
    std::fstream in(pd::g_rawfile_path, std::ios::in | std::ios::binary);

    if(!in.is_open())
    {
        ls::LOG(ls::LogLevel::WARNING) << "打开文本文件失败";
        return false;
    }

    // 读取每一个ResultData对象
    std::string line;
    while(getline(in, line))
    {
        // 构建正排索引
        struct SelectedDocInfo* s = buildForwardIndex(line);
        if(s == nullptr)
        {
            ls::LOG(ls::LogLevel::WARNING) << "构建正排索引失败";
            continue;
        }

        // 构建倒排索引
        bool flag = buildBackwardIndex(*s);

        if(!flag)
        {
            ls::LOG(ls::LogLevel::WARNING) << "构建倒排索引失败";
            continue;
        }
    }

    return true;
}

template<typename SequenceSequenceT, typename RangeT, typename PredicateT>  
SequenceSequenceT & split(SequenceSequenceT &, RangeT &&, PredicateT, 
            token_compress_mode_type = token_compress_off);

#include <boost/algorithm/string.hpp>

int main()
{
    std::string str = "aaaaa\3bbbbb\3ccccc"; // 原始数据
    std::vector<std::string> out; // 存储切割后的数据
    // 以\3作为切割符切割，默认不压缩
    boost::split(out, str, boost::is_any_of("\3"));

    std::for_each(out.begin(), out.end(), [&out](std::string s){
        std::cout << s << std::endl;
    });

    return 0;
}

输出结果：
aaaaa
bbbbb
ccccc

#include <boost/algorithm/string.hpp>

int main()
{
    std::string str = "aaaaa\3\3\3\3\3bbbbb\3ccccc"; // 原始数据
    std::vector<std::string> out; // 存储切割后的数据
    // 以\3作为切割符切割，压缩
    boost::split(out, str, boost::is_any_of("\3"), boost::token_compress_on);

    std::for_each(out.begin(), out.end(), [&out](std::string s){
        std::cout << s << std::endl;
    });

    return 0;
}

输出结果：
aaaaa
bbbbb
ccccc

int main()
{
    // ...
    boost::split(out, str, boost::is_any_of("\3"));
    // ...
}

输出结果：
aaaaa




bbbbb
ccccc

// 使用string_view自主实现split
void split(std::vector<std::string> &out, std::string_view line, std::string_view sep)
{
    if (line.empty())
        return;

    size_t pos = 0;
    size_t found;

    while ((found = line.find(sep, pos)) != std::string_view::npos)
    {
        // 添加当前位置到分隔符之间的子字符串
        out.push_back(std::string(line.substr(pos, found - pos)));
        // 更新位置到分隔符之后
        pos = found + sep.length();
    }

    // 添加最后一个分隔符之后的子字符串
    if (pos < line.length())
        out.push_back(std::string(line.substr(pos)));
}

// 频率结构
struct WordCount
{
    int title_cnt;
    int body_cnt;
};

std::unordered_map<std::string, WordCount> word_cnt_; // 词频统计

#include "include/cppjieba/Jieba.hpp"

/home/epsda/dependencies/cppjieba/include/cppjieba
├── DictTrie.hpp
├── FullSegment.hpp
├── HMMModel.hpp
├── HMMSegment.hpp
├── Jieba.hpp
├── KeywordExtractor.hpp
├── MPSegment.hpp
├── MixSegment.hpp
├── PosTagger.hpp
├── PreFilter.hpp
├── QuerySegment.hpp
├── SegmentBase.hpp
├── SegmentTagged.hpp
├── TextRankExtractor.hpp
├── Trie.hpp
├── Unicode.hpp
└── limonp // limonp文件
    ├── ArgvContext.hpp
    ├── Closure.hpp
    ├── Colors.hpp
    ├── Condition.hpp
    ├── Config.hpp
    ├── ForcePublic.hpp
    ├── LocalVector.hpp
    ├── Logging.hpp
    ├── NonCopyable.hpp
    ├── StdExtension.hpp
    └── StringUtil.hpp

#include <iostream>
#include "include/cppjieba/Jieba.hpp"

using namespace std;

int main(int argc, char **argv)
{
    cppjieba::Jieba jieba;
    vector<string> words;
    vector<cppjieba::Word> jiebawords;
    string s;
    string result;

    s = "他来到了网易杭研大厦";
    cout << s << endl;
    cout << "[demo] Cut With HMM" << endl;
    jieba.Cut(s, words, true);
    cout << limonp::Join(words.begin(), words.end(), "/") << endl;

    cout << "[demo] Cut Without HMM " << endl;
    jieba.Cut(s, words, false);
    cout << limonp::Join(words.begin(), words.end(), "/") << endl;

    s = "我来到北京清华大学";
    cout << s << endl;
    cout << "[demo] CutAll" << endl;
    jieba.CutAll(s, words);
    cout << limonp::Join(words.begin(), words.end(), "/") << endl;

    s = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造";
    cout << s << endl;
    cout << "[demo] CutForSearch" << endl;
    jieba.CutForSearch(s, words);
    cout << limonp::Join(words.begin(), words.end(), "/") << endl;

    return EXIT_SUCCESS;
}

// ...
int main(int argc, char** argv) {
    cppjieba::Jieba jieba;
    // ...
}

// ...
class Jieba {
    public:
    Jieba(const string& dict_path = "", 
            const string& model_path = "",
            const string& user_dict_path = "", 
            const string& idf_path = "", 
            const string& stop_word_path = "") 
        : dict_trie_(getPath(dict_path, "jieba.dict.utf8"), getPath(user_dict_path, "user.dict.utf8")),
        model_(getPath(model_path, "hmm_model.utf8")),
        mp_seg_(&dict_trie_),
        hmm_seg_(&model_),
        mix_seg_(&dict_trie_, &model_),
        full_seg_(&dict_trie_),
        query_seg_(&dict_trie_, &model_),
        extractor(&dict_trie_, &model_, 
                    getPath(idf_path, "idf.utf8"), 
                    getPath(stop_word_path, "stop_words.utf8")) {
    }
    // ...

    static string getPath(const string& path, const string& default_file) {
        if (path.empty()) {
            string current_dir = getCurrentDirectory();
            string parent_dir = current_dir.substr(0, current_dir.find_last_of("/\\"));
            string grandparent_dir = parent_dir.substr(0, parent_dir.find_last_of("/\\"));
            return pathJoin(pathJoin(grandparent_dir, "dict"), default_file);
        }
        return path;
    }
}

/home/epsda/dependencies/cppjieba/include
├── cppjieba // Jieba.hpp所在的路径
│   ├── DictTrie.hpp
│   ├── FullSegment.hpp
│   ├── HMMModel.hpp
│   ├── HMMSegment.hpp
│   ├── Jieba.hpp
│   ├── KeywordExtractor.hpp
│   ├── MPSegment.hpp
│   ├── MixSegment.hpp
│   ├── PosTagger.hpp
│   ├── PreFilter.hpp
│   ├── QuerySegment.hpp
│   ├── SegmentBase.hpp
│   ├── SegmentTagged.hpp
│   ├── TextRankExtractor.hpp
│   ├── Trie.hpp
│   ├── Unicode.hpp
│   └── limonp
│       ├── ArgvContext.hpp
│       ├── Closure.hpp
│       ├── Colors.hpp
│       ├── Condition.hpp
│       ├── Config.hpp
│       ├── ForcePublic.hpp
│       ├── LocalVector.hpp
│       ├── Logging.hpp
│       ├── NonCopyable.hpp
│       ├── StdExtension.hpp
│       └── StringUtil.hpp
└── dict // 与cppjieba目录同级的dict目录
    ├── README.md
    ├── hmm_model.utf8
    ├── idf.utf8
    ├── jieba.dict.utf8
    ├── pos_dict
    │   ├── char_state_tab.utf8
    │   ├── prob_emit.utf8
    │   ├── prob_start.utf8
    │   └── prob_trans.utf8
    ├── stop_words.utf8
    └── user.dict.utf8

ln -s /home/epsda/dependencies/cppjieba/dict dict

int main(int argc, char **argv)
{
    cppjieba::Jieba jieba("./dict/jieba.dict.utf8",
                          "./dict/hmm_model.utf8",
                          "./dict/user.dict.utf8",
                          "./dict/idf.utf8",
                          "./dict/stop_words.utf8");
    // ...
}

void CutForSearch(const string& sentence, vector<string>& words, bool hmm = true) const;

class SearchIndex
{
private:
    static const int title_weight_per = 10;
    static const int body_weight_per = 1;
public:
    // ...

    // 构建倒排索引
    bool buildBackwardIndex(SelectedDocInfo &sd)
    {
        // 统计标题中关键字出现的次数
        cppjieba::Jieba jieba;
        std::vector<std::string> title_words;
        jieba.CutForSearch(sd.rd.title, title_words);
        for (auto &tw : title_words)
            word_cnt_[tw].title_cnt++;

        // 统计内容中关键字出现的次数
        std::vector<std::string> body_words;
        jieba.CutForSearch(sd.rd.body, body_words);
        for (auto &bw : body_words)
            word_cnt_[bw].body_cnt++;

        // 遍历关键字哈希表获取关键字填充对应的倒排索引节点
        for (auto &word : word_cnt_)
        {
            BackwardIndexElement b;
            b.id = sd.id;
            b.word = word.first;
            // 权重统计按照公式计算
            b.weight = word.second.title_cnt * title_weight_per + word.second.body_cnt * body_weight_per;

            backward_index_[b.word].push_back(std::move(b));
        }

        return true;
    }

    // ...
private:
    // ...
    std::unordered_map<std::string, std::vector<BackwardIndexElement>> backward_index_; // 倒排索引结果
    std::unordered_map<std::string, WordCount> word_cnt_;                               // 词频统计
};

class SearchEngine
{

};

class SearchEngine
{
public:
    SearchEngine()
    {
        // 构建索引
        search_index_.buildIndex();
    }

private:
    si::SearchIndex search_index_;
};

class SearchIndex
{
private:
    // ...

    // 私有构造函数
    SearchIndex()
    {}

    // 禁用拷贝和赋值
    SearchIndex(const SearchIndex& si) = delete;
    SearchIndex& operator=(SearchIndex& si) = delete;
    SearchIndex(SearchIndex&& si) = delete;

    static SearchIndex* si;
public:
    // 获取单例对象
    static SearchIndex* getSearchIndexInstance()
    {
        if(!si)
        {
            // 加锁
            mtx.lock();
            if(!si)
                si = new SearchIndex();
            mtx.unlock();
        }

        return si;
    }

    // ...
private:
    // ...
    static std::mutex mtx;
};

SearchIndex* SearchIndex::si = nullptr;
std::mutex SearchIndex::mtx;

class SearchEngine
{
public:
    SearchEngine()
        : search_index_(si::SearchIndex::getSearchIndexInstance())
    {
        // 构建索引
        search_index_->buildIndex();
    }

    ~SearchEngine()
    {

    }

private:
    si::SearchIndex* search_index_;
};

// 根据关键字进行搜索
void search(std::string& keyword, std::string& json_string)
{

}

// 获取倒排索引结果
std::vector<BackwardIndexElement> *getBackwardIndexElement(const std::string &keyword)
{
    auto pos = backward_index_.find(keyword);
    if(pos == backward_index_.end())
    {
        ls::LOG(ls::LogLevel::WARNING) << "不存在对应的关键字";
        return nullptr;
    }

    return &backward_index_[keyword];
}

// 根据关键字进行搜索
void search(std::string &keyword, std::string &json_string)
{
    // 对用户输入的关键字进行切分
    cppjieba::Jieba jieba;
    std::vector<std::string> keywords;
    jieba.CutForSearch(keyword, keywords);

    // 查询哈希表得到结果
    std::vector<si::BackwardIndexElement> results;
    for (auto &word : keywords)
    {
        // 查倒排索引
        std::vector<si::BackwardIndexElement>* ret_ptr = search_index_->getBackwardIndexElement(word);
        if(!ret_ptr)
            continue;
        // 插入结果
        results.insert(results.end(), ret_ptr->begin(), ret_ptr->end());
    }

    // 按照权重排序
    std::stable_sort(results.begin(), results.end(), [](si::BackwardIndexElement &b1, si::BackwardIndexElement &b2)
                    { return b1.weight > b2.weight; });
}

std::unordered_map<uint64_t, si::BackwardIndexElement> select_map;

// 根据关键字进行搜索
void search(std::string &keyword, std::string &json_string)
{
    // ...
    for (auto &word : keywords)
    {
        // ...
        for(auto &bi : *ret_ptr)
        {
            // 如果文档ID已经存在，说明已经存在，否则不存在
            if(select_map.find(bi.id) == select_map.end())
                select_map[bi.id] = bi;
        }
    }

    // 遍历select_map存储结果
    for(auto &pair : select_map)
        results.push_back(std::move(pair.second));

    // ...
}

// 搜索节点结构
struct SearchIndexElement
{
    uint64_t id;
    std::vector<std::string> words;
    int weight;
};

SearchIndexElement()
    :id(0), weight(0)
{}

// std::unordered_map<uint64_t, si::BackwardIndexElement> select_map;
std::unordered_map<uint64_t, SearchIndexElement> select_map;

// 根据关键字进行搜索
void search(std::string &keyword, std::string &json_string)
{
    // ...
    for (auto &word : keywords)
    {
        // ...
        for (auto &bi : *ret_ptr)
        {
            // 如果文档ID已经存在，说明已经存在，否则不存在
            if (select_map.find(bi.id) == select_map.end())
            {
                // 获取当前文档搜索结构节点，不存在自动插入，存在直接获取
                auto &el = select_map[bi.id];
                // 如果是新节点，直接赋值；如果是重复出现的节点，覆盖
                el.id = bi.id;
                // 如果是新节点，第一次添加；如果是重复节点，追加
                el.words.push_back(bi.word);
                // 如果是新节点，直接赋值；如果是重复节点，累加
                el.weight += bi.weight;
            }
        }
    }
    // ...
}