1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
/***************************************************************************
* Copyright (C) 2004, 2005, 2006 by Stephen McInerney *
* spm@stedee.id.au *
* *
* $Id: dnshistory.h 66 2006-06-24 23:50:26Z steve $
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the *
* Free Software Foundation, Inc., *
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
* *
***************************************************************************/
/***************************************************************************
***************************************************************************
* ModificationHistory:
**********************
* 16-Mar-2005 steve Initial Creation
*
***************************************************************************
***************************************************************************/
#ifndef DNSHISTORY_DNSHISTORY_H
#define DNSHISTORY_DNSHISTORY_H 1
#include "common.h"
#include "error.h"
#include "messages.h"
#include "db_dnshistory.h"
#include "xmalloc.h"
#include "regexp.h"
/*************************************
* Simplified Error Handling MACRO's
*************************************/
/* This Macro will display errors if a given err value is non zero.
* Assumes that the message has a %d holder for the error value itself */
#define ERR_NONZERO(err, verbose, message, ...) if ((err) != 0) { ERRVPRINT ((verbose), (message), __VA_ARGS__); }
/* Similar to above, but this is a fatal condition. Display the error and exit, closing the open DB */
#define ERR_NONZERO_EXIT(err, exitcode, message, ...) if ((err) != 0) { ERRVPRINT ((VERBOSE0), (message), __VA_ARGS__); close_exit ((exitcode)); }
/* Similar to above, but instead check for a NULL value */
#define ERR_NULL_EXIT(err, exitcode, message, ...) if ((err) == NULL) { ERRVPRINT ((VERBOSE0), (message), __VA_ARGS__); close_exit ((exitcode)); }
/*************************************/
#define OVECCOUNT 300
#define BUFSIZE (128 * 2 * 512) /* size in characters of the buffer to read lines into */
#define BUFCUTOFF 125 /* When printing out long buffers, cut off here */
#define DECOMP_BUFSIZE (BUFSIZE) /* size in characters of the ZLIB buffer to read lines into */
#define MAX_RE_LENGTH 512 /* The max size in chars of an RE */
#define MAX_FILENAME_LENGTH 512 /* The max size in chars of any specified filenames */
#define PATTERN_CLF "^([^ ]+) ([^ ]+) ([^ ]+) \\[([^ ]+)"
/* We only care up to the date/time space */
#define PATTERN_COMBINED_ENHANCED "^([^ ]+) ([^ ]+) ([^ ]+) \\[([^ ]+) ([^ ]+)\\] \"([^ \"]+) ?([^ ]+)? ?([^\"]*)?\" ([^ ]+) ([^ ]+) \"(.*?)\" \"([^\"]*)\""
/* Combined Log Format, handle escaped double-quotes (APACHE) */
#define PATTERN_XFERLOG "^(... ... .. ..:..:.. ....) ([[:digit:]]+) ([[:digit:].]+)"
/* FTP/XFERLOG Log Format */
#define PATTERN_SQUID "^([[:digit:]]+)\\.([[:digit:]]+)[ ]+([[:digit:]]+) ([[:digit:].]+)"
/* SQUID Log Format. */
#define PATTERN_IPTABLES "^(... .. ..:..:..) ([^ ]+) kernel:.* SRC=([[:digit:].]+) DST=([[:digit:].]+)"
#define PATTERN_SYSLOG "^(... .. ..:..:..) ([^:]+):"
#define DATE_TIME_FORMAT "%d/%b/%Y:%H:%M:%S" /* Default DATE_TIME format for input to strptime */
#define DATE_TIME_XFERLOG_FORMAT "%a %b %d %H:%M:%S %Y"
#define DATE_TIME_IPTABLES_FORMAT "%a %d %H:%M:%S"
#define SIZE_ADDRESS (NI_MAXHOST) /* Defined in netdb.h */
#define SIZE_DATE_TIME 50
#define SIZE_URL 1024
#define SIZE_REFERER 1024
#define SIZE_BROWSER 1024
#define SIZE_COOKIE 1024
#define SIZE_CACHE (20 * 1024 * 1024)
#define THREAD_STACK_SIZE (1<<15) /* 32Kb. 14 (16K) is core'ing */
#define THREAD_MAXIMUM_THREADS 100 /* 100 threads seems a good balance. */
#define SAME_RUN 3600 /* How many seconds before we should check again that a record may have changed */
#define DNS_RESULTS_RECHECK (7 * 24 * 60 * 60) /* Only re-check DNS results every 7 days */
#define DNS_LOOKUPS 2 /* How many retries on name lookup failures */
#define DNS_RETRY_DELAY 1 /* Delay time in seconds between retries in the same run */
#define LF_NCSA_ADDRESS 1
#define LF_NCSA_USER 2
#define LF_NCSA_AUTHUSER 3
#define LF_NCSA_DATE_TIME 4
#define LF_NCSA_TIMEZONE 5
#define LF_NCSA_ACC_TYPE 6
#define LF_NCSA_URL 7
#define LF_NCSA_HTTP_VER 8
#define LF_NCSA_STATUS 9
#define LF_NCSA_BYTES 10
#define LF_NCSA_REFERER 11
#define LF_NCSA_BROWSER 12
#define LF_CUSTM_COOKIE 13
#define LF_SQUID_ADDRESS 4
#define LF_SQUID_DATE_TIME 1
#define LF_XFERLOG_ADDRESS 3
#define LF_XFERLOG_DATE_TIME 1
#define LF_IPTABLES_ADDRESS_SRC 3
#define LF_IPTABLES_ADDRESS_DST 4
#define LF_IPTABLES_DATE_TIME 1
#define DB_CACHE_SIZE 20
#define STR_NONAME "NONAME"
/* Log types */
#define LOG_UNRECOGNISED -1 /* Unrecognised log format */
#define LOG_AUTO 0 /* Try and discover the log format */
#define LOG_CLF 1 /* CLF log type */
#define LOG_FTP 2 /* wu-ftpd xferlog type */
#define LOG_SQUID 3 /* squid proxy log */
#define LOG_COMBINED 4 /* Apache Combined log type */
#define LOG_IPTABLES 5 /* syslog/linux-iptables */
/************************************************************************
* STRUCTURES *
************************************************************************/
struct dns_record_lists_t { /* linked list */
time_t date_set; /* When was 1st seen */
char fqdn[NI_MAXHOST]; /* the complete name */
struct dns_record_lists_t *next;
};
typedef struct {
struct in_addr ipaddress; /* The IP Address */
time_t date_last; /* When was last seen */
struct dns_record_lists_t *list; /* List of IP Addresses etc */
} dns_record_t;
struct buffer_position_struct {
char decomp_buf[DECOMP_BUFSIZE];
char *current_pos_ptr;
char *end_decompbuf_ptr;
};
typedef struct buffer_position_struct buffer_position;
typedef struct {
char regular_expression[MAX_RE_LENGTH]; /* Original Regular Expression */
pcre *re_pcre; /* Compiled PCRE */
int erroffset; /* Error Offset */
const char *error; /* RE error pointer, offset */
int ovector[OVECCOUNT]; /* Offset vectors */
int ret; /* Return value from various calls */
int cp_substr_ret; /* Return value from copy substring call */
} pcre_struct;
/************************************************************************
* GLOBALS *
************************************************************************/
/*-- Date/Time --*/
time_t current_day; /* The current day, check to see if we need to update a record in this run
Also see SAME_RUN */
/*-- DB Setup --*/
extern u_int32_t db_cache_size; /* Size of the DB Cache */
extern char g_db_dirfilename[MAX_FILENAME_LENGTH]; /* The name of the Database Directory Path/File to use for storage */
extern int g_dns_timeout; /* The period that DNS queries won't be retried within. Seconds */
extern char *g_filename; /* Command Line specified file to open. */
extern char *g_recombine_filename; /* Command Line specified file to open.
This file is the source of existing FQDN's
It should be a near exact copy of the STDIN or g_filename log file */
extern char *g_import_filename; /* Command Line specified file to open for Importing a dumped DB */
extern int g_verbosity; /* How many extra stuff to show. If 0, none
Bigger numbers, show more. */
extern bool g_dolookups; /* Default. Do lookups of addresses
If False then will retrieve results from DB
and output substituted log lines */
extern bool g_dotranslate; /* Do Translations */
extern bool g_dorecombine; /* Do Recombining */
extern bool g_dumpdnsdb; /* Dump the DNS History Database to STDOUT */
extern bool g_showhistory; /* Given one or more IP's show their history */
extern bool g_doimport; /* Import a Dumped DB from a File */
extern int g_dns_lookups; /* Number of attempts for each lookup if not successful */
extern unsigned int g_max_threads; /* Maximum number of threads to spawn off */
extern int g_dns_retry_delay; /* Delay time between dns lookup retries in a single run. Seconds */
extern int g_log_type; /* What type of log file is this? LOG_???? */
/************************************************************************
* FUNCTIONS *
************************************************************************/
void process_options(int, char *[]); /* Process command line options */
/************************************************************************
* MUTEXS *
************************************************************************/
extern pthread_mutex_t mutex_thread_count; /* Lock access to thread_count */
extern pthread_mutex_t mutex_db_access; /* Lock access to DB activites */
extern pthread_cond_t cond_thread_count; /* Wait for another thread to signal termination */
extern unsigned int thread_count; /* How many outsyanding threads - to know when to exit */
extern unsigned int thread_count_max; /* How many maximum number of concurrent threads */
extern unsigned int set_thread_maximum; /* Total Maximum number of concurrent threads */
extern pthread_mutex_t mutex_malloc_dns_rec;
extern int malloc_dns_rec;
extern pthread_mutex_t mutex_malloc_dns_list;
extern int malloc_dns_list;
#endif /* DNSHISTORY_DNSHISTORY_H */
/************************************************************************
************************************************************************
* END OF FILE *
************************************************************************/
|