1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/* Copyright (C) 2009-2019 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef _fsindexer_h_included_
#define _fsindexer_h_included_
#include <list>
#include <mutex>
#include "indexer.h"
#include "fstreewalk.h"
#ifdef IDX_THREADS
#include "workqueue.h"
#endif // IDX_THREADS
class FIMissingStore;
class DbUpdTask;
class InternfileTask;
namespace Rcl {
class Doc;
}
/** Index selected parts of the file system
Tree indexing: we inherits FsTreeWalkerCB so that, the processone()
method is called by the file-system tree walk code for each file and
directory. We keep all state needed while indexing, and finally call
the methods to purge the db of stale entries and create the stemming
databases.
Single file(s) indexing: there are also calls to index or purge lists of files.
No database purging or stem db updating in this case.
*/
class FsIndexer : public FsTreeWalkerCB {
public:
/** Constructor does nothing but store parameters
*
* @param cnf Configuration data
*/
FsIndexer(RclConfig *cnf, Rcl::Db *db);
virtual ~FsIndexer();
FsIndexer(const FsIndexer&) = delete;
FsIndexer& operator=(const FsIndexer&) = delete;
/**
* Top level file system tree index method for updating a given database.
*
* We open the database,
* then call a file system walk for each top-level directory.
*/
bool index(int flags);
/** Index a list of files. No db cleaning or stemdb updating */
bool indexFiles(std::list<std::string> &files, int f = ConfIndexer::IxFNone);
/** Purge a list of files. */
bool purgeFiles(std::list<std::string> &files);
/** Tree walker callback method */
FsTreeWalker::Status
processone(const std::string &fn, FsTreeWalker::CbFlag, const struct PathStat& st) override;
private:
class PurgeCandidateRecorder {
public:
PurgeCandidateRecorder()
: dorecord(false) {}
void setRecord(bool onoff) {
dorecord = onoff;
}
void record(const std::string& udi) {
// This test does not need to be protected: the value is set at
// init and never changed.
if (!dorecord)
return;
#ifdef IDX_THREADS
std::unique_lock<std::mutex> locker(mutex);
#endif
udis.push_back(udi);
}
const std::vector<std::string>& getCandidates() {
return udis;
}
private:
#ifdef IDX_THREADS
std::mutex mutex;
#endif
bool dorecord;
std::vector<std::string> udis;
};
bool launchAddOrUpdate(const std::string& udi,
const std::string& parent_udi, Rcl::Doc& doc);
FsTreeWalker m_walker;
RclConfig *m_config;
Rcl::Db *m_db;
std::string m_reason;
// Top/start directories list
std::vector<std::string> m_tdl;
// Store for missing filters and associated mime types
FIMissingStore *m_missing;
// Recorder for files that may need subdoc purging.
PurgeCandidateRecorder m_purgeCandidates;
// The configuration can set attribute fields to be inherited by
// all files in a file system area. Ie: set "rclaptg = thunderbird"
// inside ~/.thunderbird. The boolean is set at init to avoid
// further wasteful processing if no local fields are set.
// This should probably moved to internfile so that the
// localfields get exactly the same processing as those generated by the
// filters (as was done for metadatacmds fields)
bool m_havelocalfields;
std::string m_slocalfields;
std::map<std::string, std::string> m_localfields;
// Activate detection of xattr-only document updates. Experimental, so
// needs a config option
bool m_detectxattronly;
// No retry of previously failed files
bool m_noretryfailed;
// use FADV_DONTNEED if available
bool m_cleancache{false};
#ifdef IDX_THREADS
friend void *FsIndexerDbUpdWorker(void*);
friend void *FsIndexerInternfileWorker(void*);
WorkQueue<InternfileTask*> m_iwqueue;
WorkQueue<DbUpdTask*> m_dwqueue;
bool m_haveInternQ;
bool m_haveSplitQ;
RclConfig *m_stableconfig;
#endif // IDX_THREADS
bool init();
void localfieldsfromconf();
void setlocalfields(const std::map<std::string, std::string>& flds, Rcl::Doc& doc);
std::string getDbDir() {return m_config->getDbDir();}
FsTreeWalker::Status
processonefile(RclConfig *config, const std::string &fn,
const struct PathStat &, const std::map<std::string, std::string>& localfields);
void shutdownQueues(bool);
};
#endif /* _fsindexer_h_included_ */
|