I use opencv and surf descriptor, for each image i get a descriptor matrix, then I create a big matrix (Mat) for use flann index search. the problem is that when big matrix size grow (over 30 gb) I will not have enough free memory. now I serialize matrix to binary file, and I have one solution: I limite max size and split it into several files, so after every search operation, I read files by queue and then combine the results. is there more effincey solution to read it once?
//code
   class FlannIndexModel
{
    cv::Ptr<cv::flann::Index> flannIndex;
    cv::Mat dbDescs;
    string fileName;
public:
    vector<IndecesMappingModel> imap;
    FlannIndexModel(string fileName)
    {
        this->fileName = fileName;
        flannIndex = new cv::flann::Index();
    }
    size_t Size()
    {
        size_t sizeInBytes = dbDescs.total() * dbDescs.elemSize();
        return sizeInBytes/1000000;
    }
    void Load()
    {
        FileManager::LoadMat(dbDescs, (fileName + "_desc.bin"));
        FileManager::LoadImap(imap, (fileName + "_imap.bin"));
        flannIndex->load(dbDescs, (fileName + "_flann.bin"));
        cout << " Flann Load: " << " dbDescs rows= " << dbDescs.rows << " imap= " << imap.size() << endl;
    }
    void Save()
    {
        FileManager::SaveMat(dbDescs, (fileName + "_desc.bin"));
        FileManager::SaveImap(imap, (fileName + "_imap.bin"));
        flannIndex->save((fileName + "_flann.bin"));
    }
    void Add(vector<ImageDescModel> imges)
    {
        vector<cv::Mat> descs;
        int r = dbDescs.rows;
        for (int i = 0; i < imges.size(); i++)
        {
            auto desc = imges[i].Desc;
            if (desc.empty())
                continue;
            descs.push_back(desc);
            imap.push_back(IndecesMappingModel(imges[i].FileName, r, r + desc.rows - 1));
            r += desc.rows;
        }
        if (!dbDescs.empty())
            descs.push_back(dbDescs);
        vconcat(descs, dbDescs);
    }
    void Calcul()
    {
        flannIndex->build(dbDescs, cv::flann::KDTreeIndexParams::KDTreeIndexParams(4));
    }
    vector<IndecesMappingModel> Search(cv::Mat queryDescriptors, int num)
    {
        for (auto &img : imap)
        {
            img.Similarity = 0;
        }
        cv::Mat indices(queryDescriptors.rows, 2, CV_32S);
        cv::Mat dists(queryDescriptors.rows, 2, CV_32F);
        flannIndex->knnSearch(queryDescriptors, indices, dists, 2, cv::flann::SearchParams(24));
#pragma omp for
        for (int i = 0; i < indices.rows; i++)
        {
            if (dists.at<float>(i, 0) < (0.6 * dists.at<float>(i, 1)))
            {
                for (auto &img : imap)
                {
                    if (img.IndexStart <= indices.at<int>(i, 0) && img.IndexEnd >= indices.at<int>(i, 0))
                    {
                        img.Similarity++;
                        break;
                    }
                }
            }
        }
        std::sort(imap.begin(), imap.end());
        if (imap.size() > num)
        {
            vector<IndecesMappingModel> result(imap.begin(), imap.begin() + num);
            return result;
        }
        else
        {
            return imap;
        }
    }
};
