/***************************************************************************
 *   Copyright (C) 2004, 2005, 2006 by Stephen McInerney                   *
 *   spm@stedee.id.au                                                      *
 *                                                                         *
 *   $Id: dnshistory.c 66 2006-06-24 23:50:26Z steve $
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 *                                                                         *
 ***************************************************************************/

/***************************************************************************
 * dnshistory
 *
 * a web logfile preparser for extracting IP addresses and doing lookups
 *
 ***************************************************************************/

/***************************************************************************
 * PURPOSE:
 *
 * Given a log file (stdin or cmdline) parse the address field and record
 * a history of DNS information and changes over time
 * Time used is _current_ time NOT log time
 *
 * Expects addresses to be solely in IP format (xxx.yyy.zzz.aaa)
 * Stores the results in a Berkely DB
 * Requires PCRE to extract the address - a bit over kill, but more
 *   flexible over the longer term
 *
 * Files specified as arguments can be in .gz format
 *
 * Operates in one of SIX modes:
 * 1.  Given a logfile, lookup IP addresses and store FQD names in a Berkeley DB
 * 2.  Given two log files:
 *        One with IP Address
 *        Other being same log but with IP Address replaced with FQD Names
 *      Store the IP Address/FQDN pairs in a Berkeley DB
 *      This option assumes #1 was done some time in the past, and we now
 *       wish to store the values in this Berkeley DB. eg via dnstran
 * 3.  Given a logfile, retrieve FQD names from a Berkely DB
 *      Replacing the IP Address from the Log with the FQDN;
 *        display the now altered log line
 * 4.  Dump the DB out as a simple text file
 *      a "NONAME" fqdn implies a failure to lookup a name, conversely
 *      the IP address as the fqdn implies a successful failure to get a name
 * 5.  Read same back in
 * 6.  Do history retrievals from CMD line IP Addresses
 *
 ***************************************************************************/

/***************************************************************************
 ***************************************************************************
 * Logic:
 *
 * Case 1:
 *   Grab and Hold the current time
 *   Read a log file
 *   Extract the address from each line
 *   Check for last lookup time on this record
 *       - use the 'current time' as the basis for comparison
 *     If appropriate, fire off a new thread to deal with the name resolution
 *       Each individual thread will store the updated result in a Berkeley DB
 *
 * Case 2:
 *   Read the RAW Log (has no lookups yet done)
 *   Read the converted Log
 *   Grab the time from the log entry
 *   Check for last lookup time on this record
 *       - use the log time as the basis for comparison
 *     If appropriate, store a new record with a new time
 *
 * Case 3:
 *   Read the supplied log
 *   Lookup the IP Address from a previous DB
 *   Replace the IP Address with the retrieved FQDN and print to STDOUT
 *     If no value, simply send the IP Address
 *   Print to STDOUT the rest of the logline
 *   
 * Case 4:
 *   Cursor thru the supplied DB (or use default)
 *   Display the IP Address and each time/fqdn stored value
 *       for this IP Address on a single line
 *     tab separated list between time/fqdn pairs
 *     comma separated between time and fqdn values
 *
 * Case 5:
 *   Open the import file
 *   Parse the import file, record chunk by chunk
 *     Create a normal "dnshistory" structure/linked list
 *     Store via the normal method
 *
 * Case 6:
 *   Given one or more IP Address on the command line
 *   Display the IP Address and each time/fqdn stored value
 *       for this IP Address on a single line
 *     tab separated list between time/fqdn pairs
 *     comma separated between time and fqdn values
 *
 *
 * IP Addresses are stored as in_addr - see netinet/in.h (not char[15])
 *
 ***************************************************************************
 ***************************************************************************/

/***************************************************************************
 ***************************************************************************
 * TODO:
 * 
 ***************************************************************************
 ***************************************************************************/

/***************************************************************************
 ***************************************************************************
 * BUGS:
 * +    Expects addresses to be solely in IP4 format (xxx.yyy.zzz.aaa)
 * +    Doesn't check for already converted logs
 * +    Can't deal with log lines with escaped quotes: \"
 *
 ***************************************************************************
 ***************************************************************************/

/***************************************************************************
 ***************************************************************************
 * ModificationHistory:
 **********************
 * 11-Jun-2004 steve    Initial Creation; modified from visitors.c
 * 18-Mar-2005 steve    Modularise and add output ability
 * 12-Aug-2005 steve    flag getnameinfo to error on no resolution
 * 10-Oct-2005 steve    Add showhistory
 *                      Refactor for display output
 * 24-Dec-2005 steve    Open DB's as RO or RW as appropriate
 * 02-Jan-2006 steve    Add import (Case 5)
 *
 ***************************************************************************
 * CODING STYLE:
 * indent -kr -bad -bap -bbo -nbc -br -brs -c49 -cd49  -i4 -ce -cp17 -l180 -nut -npcs -psl
 ***************************************************************************
 ***************************************************************************/


#include "dnshistory.h"

/************************************************************************
 *                              GLOBALS                                 *
 ************************************************************************/
/*-- Date/Time --*/
time_t current_day;                             /* The current day, check to see if we need to update a record in this run
                                                   Also see SAME_RUN */

/*-- DB Setup --*/
DB *dnshistory_db_ptr;                          /* Database Pointer */
char g_db_dirfilename[MAX_FILENAME_LENGTH] = DATABASE;  /* File name for the Database */

int rtn_db;                                     /* Value from DB calls; Used for program exit if necessary */
unsigned long int number_added_addresses = 0;   /* How many Addresses did we save/store? */
unsigned long int number_name_lookups = 0;      /* How many Name Lookups did we perform? */
unsigned long int number_retries = 0;           /* How many name lookup retries */
unsigned long int number_successful_retries = 0;        /* How many name lookup retries */
unsigned long int number_successful_fqdns = 0;  /* How many FQDNs did we store, vs just IP's */
unsigned long int total_lines = 0;              /* How many log lines in total */
unsigned long int bad_lines = 0;                /* How many dud log lines did we get? */
unsigned long int bad_recom_lines = 0;          /* How many dud recombined log lines did we get? */
unsigned long int bad_mismatched_lines = 0;     /* How many mismatched lines did we get? */

pcre *cmp_log_regexp = NULL;                    /* Main compiled RE - use as pointer only to one of the below */
pcre *cmp_log_regexp_clf = NULL;                /* CLF compiled RE */
pcre *cmp_log_regexp_xferlog = NULL;            /* FTP, xferlog format compiled RE */
pcre *cmp_log_regexp_squid = NULL;              /* SQUID format compiled RE */
pcre *cmp_log_regexp_combined_enhanced = NULL;  /* Enhanced Combined compiled RE */
pcre *cmp_log_regexp_iptables = NULL;           /* syslog/linux-iptables compiled RE */
pcre *cmp_log_regexp_syslog = NULL;             /* syslog compiled RE - used to test for possible iptables*/

int g_log_type = 0;                               /* What type of log file is this? LOG_???? */

/************************************************************************
 *                              FUNCTIONS                               *
 ************************************************************************/

/* These two are the main worker functions - call as new threads */
void *add_new_address(void *);                  /* Haven't seen this address before. Do the name lookup; Add a new entry. */
void *add_old_address(void *);                  /* Have! seen this address before. Do the name lookup; Update the old entry */

void store_dns_records(dns_record_t *);         /* Store a dns record in our DB */

void add_address(char *);                       /* Given an IP address; create a new thread to do a lookup/DB Update */
void retrieve_address(char *, char *, char *);  /* Retrieve the FQDN from the DB */
void add_recombined_address(char *, char *, char *);

void check_n_fix_fqdn(char *);                  /* strip any funky chars from a FQDN */
int name_lookup(struct in_addr ipaddr, char *); /* Do the actual name lookup */
int address_exists(dns_record_t *);             /* Does this IP address already exist, looked up? */
void name_lookup_errors(int);                   /* Deal with name lookup errors */
void drop_long_lines(gzFile *, char *, buffer_position *);      /* Strip Log Lines that are too big */
char *get_log_line(char *, int, gzFile *, buffer_position *);   /* Faster version of gzgets */

void dump_dns_historydb(DB **);                 /* Dump the DB out */
void showhistory(DB **, int argc, char *[]);
void import_dns_historydb(void);

void increase_thread_counter(void);
void decrease_thread_counter(void);

void close_exit(int);

void re_compile_all_regexes(void);
int identify_log_format(char *);

/************************************************************************
 *                              MUTEXS                                  *
 ************************************************************************/
pthread_mutex_t mutex_thread_count = PTHREAD_MUTEX_INITIALIZER; /* Lock access to thread_count */
unsigned int thread_count = 0;                  /* How many outstanding threads - to know when to exit */
unsigned int thread_count_max = 0;              /* How many maximum number of concurrent threads */
pthread_cond_t cond_thread_count = PTHREAD_COND_INITIALIZER;    /* Wait for another thread to signal termination */

pthread_mutex_t mutex_db_access = PTHREAD_MUTEX_INITIALIZER;    /* Lock access to DB activites */
pthread_mutex_t mutex_malloc_dns_rec = PTHREAD_MUTEX_INITIALIZER;
int malloc_dns_rec = 0;
pthread_mutex_t mutex_malloc_dns_list = PTHREAD_MUTEX_INITIALIZER;
int malloc_dns_list = 0;

pthread_mutex_t mutex_other_counters = PTHREAD_MUTEX_INITIALIZER;       /* Lock access to general counters  */

/************************************************************************
 ************************************************************************
 ***                            MAIN                                  ***
 ************************************************************************
 ************************************************************************/
/* MAIN */
int
main(int argc, char *argv[])
{
    /**********************************************************************/
    pcre_struct main_pcre;
    pcre_struct recombine_pcre;

    char buffer_primary[BUFSIZE];               /* Primary log buffer */
    int buffer_length;                          /* How many char's currently in the Primary Buffer */
    char buffer_recombine[BUFSIZE];             /* Recombine log buffer */
    int buffer_recombine_length;                /* How many char's currently in the Recombine Buffer */
    char buffer_tmp_output[BUFSIZE];            /* Temporary buffer for output displaying */
    
    bool have_set_logtype_flag = false;             /* Set to true when we've set the log type */

    /* Log File handlers */
    gzFile *file_input = NULL;                  /* Input File descriptor */
    gzFile *file_recombine_input = NULL;        /* Recombine Input File descriptor */
    char *fgets_rtn = NULL;                     /* Return value from doing fgets. Check for end of file */

    char str_address[NI_MAXHOST];               /* IP Addresses */
    char str_address2[NI_MAXHOST];              /* IP Addresses for iptables lookups */
    char str_previous_address[NI_MAXHOST];      /* Previous IP Addresses */
    char str_fqdn[NI_MAXHOST] = "";                  /* Retrieved FQDN Address */
    char str_fqdn2[NI_MAXHOST] = "";                 /* Retrieved FQDN Address for iptables lookups */
    char *buf_ptr;                              /* Offset pointer into the file's buffer - used to print from other than the start */

    char str_time[SIZE_DATE_TIME];              /* String to hold the current time out of the current log line */
    char str_time_raw[SIZE_DATE_TIME];
    char access_size_raw[25];
    char access_size_recombine[25];
    int comp_ret = 0;

    /* Recombine Variables */
    bool badlogline_flag = false;               /* Used to indictate we need to "continue" the main loop due to a long line error */
    char str_address_recombine[NI_MAXHOST];     /* Recombined Addresses - should be FQDN or raw IP Address */
    char *bufer_recombine_ptr;                  /* Offset pointer into the recombine file's buffer - used to print from other than the start */

    buffer_position buf_posn;
    buffer_position buf_recombine_posn;

    unsigned int position_address = 0;
    unsigned int position_datetime = 0;

    /**********************************************************************/

    /*****************************
     *          BEGIN
     *****************************/
    process_options(argc, argv);

    /* Dumping the Database is easy
     * Do so and exit */
    if (g_dumpdnsdb == true) {
        open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
        dump_dns_historydb(&dnshistory_db_ptr);
        close_exit(V_EXIT_OK);
    }
    /* Likewise with individual IP lookups */
    if (g_showhistory == true) {
        open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
        showhistory(&dnshistory_db_ptr, argc, argv);
        close_exit(V_EXIT_OK);
    }
    /* Do the import and exit if chosen */
    if (g_doimport == true) {
        open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_CREATE | DB_EXCL);
        import_dns_historydb();
        close_exit(V_EXIT_OK);
    }
    if (g_dotranslate == true) {
        open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_RDONLY);
    } else {
        open_dnshistory_db(&dnshistory_db_ptr, g_db_dirfilename, db_cache_size, DB_CREATE);
    }


    /* From here on, we're looping over log files */
    VPRINT(VERBOSE4, "Setting pthread stacksize to: %d\n", THREAD_STACK_SIZE);

    re_compile_all_regexes();

    buf_posn.current_pos_ptr = buf_posn.decomp_buf + DECOMP_BUFSIZE;
    buf_posn.end_decompbuf_ptr = NULL;

    /* Open Input Log File if appropriate */
    if (g_filename != NULL) {
        VPRINT(VERBOSE4, "Using file: %s\n", g_filename);
        /* use_file = true; */
        file_input = gzopen(g_filename, "rb");
        ERR_NULL_EXIT(file_input, V_EXIT_BAD_FILE_OPEN, msg_F_file_open, g_filename);
    }

    /* Compile additional, detailed PCRE for line checking if doing recombining */
    /* Open Input Recombine Log File if appropriate */
    if (g_dorecombine == true) {
        memset(&recombine_pcre, 0, sizeof(recombine_pcre));
        strncpy(recombine_pcre.regular_expression, PATTERN_COMBINED_ENHANCED, MAX_RE_LENGTH);
        VPRINT(VERBOSE3, "Recombine Reg Ex Pattern: %s\n", recombine_pcre.regular_expression);
        recombine_pcre.re_pcre = pcre_compile(recombine_pcre.regular_expression, 0, &recombine_pcre.error, &recombine_pcre.erroffset, NULL);
        if (main_pcre.re_pcre == NULL) {
            re_compile_failed(recombine_pcre.erroffset, recombine_pcre.error, recombine_pcre.regular_expression);
        }

        buf_recombine_posn.current_pos_ptr = buf_recombine_posn.decomp_buf + DECOMP_BUFSIZE;
        buf_recombine_posn.end_decompbuf_ptr = NULL;

        VPRINT(VERBOSE4, "Using Recombine file: %s\n", g_recombine_filename);
        file_recombine_input = gzopen(g_recombine_filename, "rb");
        ERR_NULL_EXIT(file_recombine_input, V_EXIT_BAD_FILE_OPEN, msg_F_file_open, g_recombine_filename);
    }


    current_day = time(NULL);                   /* set the current time - use for checking if a record is current to this run or not */
    str_previous_address[0] = '\0';


    /*************************************************************
     * The Primary Loop!
     *************************************************************/
    VPRINT(VERBOSE4, "Beginning Main Loop.%s\n", "");
    while (1) {
        /* Read a line from file or stdin */
        if (file_input != NULL) {
            fgets_rtn = get_log_line(buffer_primary, BUFSIZE, file_input, &buf_posn);
        } else {
            fgets_rtn = fgets(buffer_primary, BUFSIZE, stdin);
        }
        if (fgets_rtn == NULL) {
            /* At file end! */
            break;
        }

        total_lines++;

        /* check for log line too long */
        buffer_length = (int) strlen(buffer_primary);
        if (strpbrk("\n", buffer_primary) == NULL) {
            if (file_input == NULL) {
                drop_long_lines(NULL, buffer_primary, NULL);
            } else {
                drop_long_lines(file_input, buffer_primary, &buf_posn);
            }
            bad_lines++;
            if (g_dorecombine == true) {
                badlogline_flag = true;
            } else {
                continue;
            }

        }

        /* Identify the log type on first run thru */
        if (! have_set_logtype_flag) {
            if (g_log_type == LOG_AUTO) {
                g_log_type = identify_log_format(buffer_primary);
                if (g_log_type <= 0) {
                    ERRVPRINT(VERBOSE0, "Cannot recognise log format.%s", "\n");
                    exit(1);
                }
            }
            memset(&main_pcre, 0, sizeof(main_pcre));
            switch (g_log_type) {
            case LOG_FTP:
                main_pcre.re_pcre = cmp_log_regexp_xferlog;
                strncpy(main_pcre.regular_expression, PATTERN_XFERLOG, MAX_RE_LENGTH);
                position_address = LF_XFERLOG_ADDRESS;
                position_datetime = LF_XFERLOG_DATE_TIME;
                break;
            case LOG_SQUID:
                main_pcre.re_pcre = cmp_log_regexp_squid;
                strncpy(main_pcre.regular_expression, PATTERN_SQUID, MAX_RE_LENGTH);
                position_address = LF_SQUID_ADDRESS;
                position_datetime = LF_SQUID_DATE_TIME;
                break;
            case LOG_CLF:
                main_pcre.re_pcre = cmp_log_regexp_clf;
                strncpy(main_pcre.regular_expression, PATTERN_CLF, MAX_RE_LENGTH);
                position_address = LF_NCSA_ADDRESS;
                position_datetime = LF_NCSA_DATE_TIME;
                break;
            case LOG_IPTABLES:
                main_pcre.re_pcre = cmp_log_regexp_iptables;
                strncpy(main_pcre.regular_expression, PATTERN_IPTABLES, MAX_RE_LENGTH);
                position_address = LF_IPTABLES_ADDRESS_SRC;
                position_datetime = LF_IPTABLES_DATE_TIME;
                break;
            default:
                ERRVPRINT(VERBOSE0, "Unknown LOG Type Setting. Sorry.... : %d\n", g_log_type);
                exit(1);
            }
            VPRINT(VERBOSE3, "Main Reg Ex Pattern: %s\n", main_pcre.regular_expression);
        }


        if (badlogline_flag != true) {
            VPRINT(VERBOSE3, "-%lu-%s", total_lines, buffer_primary);

            /* Apply the pattern match. */
            main_pcre.ret = pcre_exec(main_pcre.re_pcre, NULL, buffer_primary, buffer_length, 0, 0, main_pcre.ovector, OVECCOUNT);
            /* check for RE matching errors */
            if (main_pcre.ret <= 0) {
                if (g_log_type != LOG_IPTABLES) {
                    re_check_errors(main_pcre.ret, total_lines, buffer_primary);
                    if (g_dorecombine == true) {
                        badlogline_flag = true;
                    } else {
                        continue;
                    }
                } else {
                    /* Assume no RegEx failures with iptables. Ha! Idea is to skip over other syslog messages */
                    continue;
                }
            }

            main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret, position_address, str_address, BUFSIZE);
            if (main_pcre.cp_substr_ret < 0) {
                error_substring_extract(main_pcre.regular_expression, buffer_primary, position_address, main_pcre.cp_substr_ret, total_lines);
                if (g_dorecombine == true) {
                    badlogline_flag = true;
                } else {
                    continue;
                }
            }
            if (g_log_type == LOG_IPTABLES) {
                main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret,LF_IPTABLES_ADDRESS_DST , str_address2, BUFSIZE);
                if (main_pcre.cp_substr_ret < 0) {
                    error_substring_extract(main_pcre.regular_expression, buffer_primary, position_address, main_pcre.cp_substr_ret, total_lines);
                    if (g_dorecombine == true) {
                        badlogline_flag = true;
                    } else {
                        continue;
                    }
                }
            }
        }

        if (g_dotranslate == true) {
            /*************************************************************
             * Do the translating
             *************************************************************/
            /* Retrieve correct fqdn from DB by date match.
             * Spit out full line with replaced IP Address */
            comp_ret = strncmp(str_previous_address, str_address, SIZE_ADDRESS);
            if (comp_ret != 0) {
                /* Only update the FQDN if the most recent address has changed */

                /* Extract Date/Time */
                main_pcre.cp_substr_ret = pcre_copy_substring(buffer_primary, main_pcre.ovector, main_pcre.ret, position_datetime, str_time, SIZE_DATE_TIME);
                if (main_pcre.cp_substr_ret < 0) {
                    ERRVPRINT(VERBOSE0, msg_F_vital_substring, position_datetime);
                    close_exit(V_EXIT_PCRE_NO_SUBSTR);
                }

                retrieve_address(str_address, str_fqdn, str_time);
                strncpy(str_previous_address, str_address, SIZE_ADDRESS);
                
                if (g_log_type == LOG_IPTABLES) {
                    retrieve_address(str_address2, str_fqdn2, str_time);
                }
            }
            if (g_log_type == LOG_IPTABLES) {
                if ((str_fqdn[0] == '\0') && (str_fqdn2[0] == '\0')) {
                    /* No change, display line-in as line-out */
                    printf("%s", buffer_primary);
                } else {
                    if (str_fqdn[0] == '\0') {
                        buffer_tmp_output[0] = '\0';
                        strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) - 1) + 1]);
                        printf("%s", buffer_tmp_output);
                        printf("%s", str_fqdn2);
                        buffer_tmp_output[0] = '\0';
                        strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) + 1)]);
                        printf("%s", buffer_tmp_output);
                    } else {
                        buffer_tmp_output[0] = '\0';
                        strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((position_address * 2) - 1) + 1]);
                        printf("%s", buffer_tmp_output);
                        printf("%s", str_fqdn);
                        printf(" DST=");
                        buffer_tmp_output[0] = '\0';
                        if (str_fqdn2[0] == '\0') {
                            strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[(LF_IPTABLES_ADDRESS_DST * 2)]);
                        } else {
                            printf("%s", str_fqdn2);
                            strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((LF_IPTABLES_ADDRESS_DST * 2) + 1)]);
                        }
                        printf("%s", buffer_tmp_output);
                    }
                }
            } else {
                if (str_fqdn[0] == '\0') {
                    /* No change, display line-in as line-out */
                    printf("%s", buffer_primary);
                } else {
                    /* Have a returned value. Displayed FQDN and rest of line minus IP Address. */
                    /* print before */
                    if (position_address > 1) {
                        buffer_tmp_output[0] = '\0';
                        strncat(buffer_tmp_output, buffer_primary, main_pcre.ovector[((position_address * 2) - 1) + 1]);
                        printf("%s", buffer_tmp_output);
                    }
                    printf("%s", str_fqdn);
                    buffer_tmp_output[0] = '\0';
                    strcat(buffer_tmp_output, buffer_primary + main_pcre.ovector[((position_address * 2) + 1)]);
                    printf("%s", buffer_tmp_output);
                }
            }
        } else if (g_dorecombine == true) {
            if (g_log_type != LOG_CLF) {
                ERRVPRINT(VERBOSE0, "Sorry, can't recombine non CLF logs. Yet. Please email author!%s", "\n");
                exit(1);
            }
            /*************************************************************
             * Do the recombining
             *************************************************************/
            /* Grab part of the recombine log file */
            fgets_rtn = get_log_line(buffer_recombine, BUFSIZE, file_recombine_input, &buf_recombine_posn);
            ERR_NULL_EXIT(fgets_rtn, V_EXIT_EARLY_LOG_CLOSE, msg_F_early_log_termination, total_lines);

            buffer_recombine_length = (int) strlen(buffer_recombine);
            /* check for log line too long */
            if ((strpbrk("\n", buffer_recombine) == NULL) || (badlogline_flag == true)) {
                drop_long_lines(file_recombine_input, buffer_recombine, &buf_recombine_posn);
                if (badlogline_flag == false) {
                    bad_lines++;
                }
                badlogline_flag = false;
                continue;
            }

            VPRINT(VERBOSE3, "+%lu+%s", total_lines, buffer_recombine);
            VPRINT(VERBOSE5, "  New Address: %s  Old Address: %s\n", str_address, str_previous_address);
            if (strncmp(str_previous_address, str_address, SIZE_ADDRESS) != 0) {
                /* Only update the FQDN if the most recent address has changed */

                strncpy(str_previous_address, str_address, SIZE_ADDRESS);

                main_pcre.ret = pcre_exec(main_pcre.re_pcre, NULL, buffer_recombine, buffer_recombine_length, 0, 0, main_pcre.ovector, OVECCOUNT);
                if (main_pcre.ret <= 0) {
                    re_check_errors(main_pcre.ret, total_lines, buffer_recombine);
                    bad_lines++;
                    continue;
                }

                main_pcre.cp_substr_ret =
                    pcre_copy_substring(buffer_recombine, main_pcre.ovector, main_pcre.ret, LF_NCSA_ADDRESS, str_address_recombine, sizeof(str_address_recombine));
                if (recombine_pcre.cp_substr_ret < 0) {
                    error_substring_extract(recombine_pcre.regular_expression, buffer_recombine, LF_NCSA_ADDRESS, recombine_pcre.cp_substr_ret, total_lines);
                    bad_lines++;
                    continue;
                }

                /* Extract Date/Time */
                main_pcre.cp_substr_ret = pcre_copy_substring(buffer_recombine, main_pcre.ovector, main_pcre.ret, LF_NCSA_DATE_TIME, str_time, sizeof(str_time));
                if (recombine_pcre.cp_substr_ret < 0) {
                    /* Fatal Error, as we've successfully matched elsewhere. We really shouldn't ever get here... */
                    ERRVPRINT(VERBOSE0, "%s\n", msg_F_vital_substring);
                    close_exit(V_EXIT_PCRE_NO_SUBSTR);
                }

                buf_ptr = strpbrk(buffer_primary, " ");
                bufer_recombine_ptr = strpbrk(buffer_recombine, " ");

                comp_ret = strncmp(buf_ptr, bufer_recombine_ptr, BUFSIZE);
                if (comp_ret != 0) {
                    /* We have a mismatched line.
                     *  Now verify if truely borked file(s) or simply an IP Address substitution elsewhere in a line */

                    /* First off, we will compare the date/time stamps */
                    /* We already have the recombine buffer's date time, so grab from the "raw" logfile */
                    recombine_pcre.ret = pcre_exec(recombine_pcre.re_pcre, NULL, buffer_primary, buffer_length, 0, 0, recombine_pcre.ovector, OVECCOUNT);
                    if (recombine_pcre.ret <= 0) {
                        re_check_errors(recombine_pcre.ret, total_lines, buffer_primary);
                        bad_lines++;
                        continue;
                    }
                    recombine_pcre.cp_substr_ret =
                        pcre_copy_substring(buffer_primary, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_DATE_TIME, str_time_raw, sizeof(str_time_raw));
                    if (recombine_pcre.cp_substr_ret < 0) {
                        error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_DATE_TIME, recombine_pcre.cp_substr_ret, total_lines);
                        bad_lines++;
                        continue;
                    }
                    comp_ret = strncmp(str_time_raw, str_time, strlen(str_time));
                    ERR_NONZERO_EXIT(comp_ret, V_EXIT_MISMATCHED_LINES, msg_F_mismatched_lines, total_lines, buffer_primary, buffer_recombine);

                    /* Date/Time ok. Lets look at size in bytes. Would use URL, but they get replaced as well unf. */
                    /* Copy out the size in bytes of the access - already looking at the main log */
                    recombine_pcre.cp_substr_ret =
                        pcre_copy_substring(buffer_primary, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_BYTES, access_size_raw, sizeof(access_size_raw));
                    if (recombine_pcre.cp_substr_ret < 0) {
                        error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_BYTES, recombine_pcre.cp_substr_ret, total_lines);
                        bad_lines++;
                        continue;
                    }
                    /* Now from the recombine file/buffer */
                    recombine_pcre.ret = pcre_exec(recombine_pcre.re_pcre, NULL, buffer_recombine, buffer_recombine_length, 0, 0, recombine_pcre.ovector, OVECCOUNT);
                    if (recombine_pcre.ret <= 0) {
                        re_check_errors(recombine_pcre.ret, total_lines, buffer_recombine);
                        bad_lines++;
                        continue;
                    }
                    recombine_pcre.cp_substr_ret =
                        pcre_copy_substring(buffer_recombine, recombine_pcre.ovector, recombine_pcre.ret, LF_NCSA_BYTES, access_size_recombine, sizeof(access_size_recombine));
                    if (recombine_pcre.cp_substr_ret < 0) {
                        error_substring_extract(recombine_pcre.regular_expression, buffer_primary, LF_NCSA_DATE_TIME, recombine_pcre.cp_substr_ret, total_lines);
                        bad_lines++;
                        continue;
                    }
                    comp_ret = strncmp(access_size_raw, access_size_recombine, strlen(access_size_raw));
                    ERR_NONZERO_EXIT(comp_ret, V_EXIT_MISMATCHED_LINES, msg_F_mismatched_lines, total_lines, buffer_primary, buffer_recombine);

                    ERRVPRINT(VERBOSE2, msg_W_mismatched_lines, total_lines, buffer_primary, buffer_recombine);
                    bad_mismatched_lines++;
                }

                add_recombined_address(str_address, str_address_recombine, str_time);
            }
        } else {                                /* Default ==> (g_dolookups == true) */
            /*************************************************************
             * Default Action - do the lookups and store results in DB
             *************************************************************/

            /* Rapid check: have we seen this IP before?
             *  Typically we expect to see multiple entries to the same address, one after the other
             * If not, add the new one, and update the old address */
            comp_ret = strncmp(str_previous_address, str_address, SIZE_ADDRESS);
            if (comp_ret != 0) {
                /* Don't exceed Maximum Number of threads! Stop and hold, once we get too many */
                pthread_mutex_lock(&mutex_thread_count);
                while (thread_count >= g_max_threads) {
                    VPRINT(VERBOSE2, "HOLDING: Exceeding Maximum Thread Count!: %d <= %d\n", g_max_threads, thread_count);
                    pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
                }
                pthread_mutex_unlock(&mutex_thread_count);
                add_address(str_address);
                strncpy(str_previous_address, str_address, SIZE_ADDRESS);
            }
            if (g_log_type == LOG_IPTABLES) {
                /* Don't exceed Maximum Number of threads! Stop and hold, once we get too many */
                pthread_mutex_lock(&mutex_thread_count);
                while (thread_count >= g_max_threads) {
                    VPRINT(VERBOSE2, "HOLDING: Exceeding Maximum Thread Count!: %d <= %d\n", g_max_threads, thread_count);
                    pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
                }
                pthread_mutex_unlock(&mutex_thread_count);
                add_address(str_address2);
            }
        }
    }


    if (g_dolookups == true) {
        pthread_mutex_lock(&mutex_thread_count);
        while (thread_count > 0) {
            VPRINT(VERBOSE5, "FINISHING: Countdown thread count: %d\n", thread_count);
            pthread_cond_wait(&cond_thread_count, &mutex_thread_count);
        }

        VPRINT(VERBOSE1, "Maximum Concurrent Threads:  %d\n", thread_count_max);
        VPRINT(VERBOSE5, "Final Remaining DNS Records: %d\n", malloc_dns_rec);
        VPRINT(VERBOSE5, "Final Remaining DNS Lists:   %d\n", malloc_dns_list);
    }

    VPRINT(VERBOSE1, "Stored %lu Addresses from %lu Log Lines\n", number_added_addresses, total_lines);
    if (number_name_lookups > 0) {
        VPRINT(VERBOSE1, "Number of Name Lookups Performed: %lu of %lu Log Lines\n", number_name_lookups, total_lines);
    }
    if (bad_lines > 0) {
        VPRINT(VERBOSE0, "%s%lu of %lu\n", msg_I_number_bad_lines, bad_lines, total_lines);
    }
    if (bad_recom_lines > 0) {
        VPRINT(VERBOSE0, "%s%lu of %lu\n", "Number of Bad Recombined Lines: ", bad_recom_lines, total_lines);
    }
    if (bad_mismatched_lines > 0) {
        VPRINT(VERBOSE0, "%s%lu of %lu\n", "Number of Possibly Mismatched Lines: ", bad_mismatched_lines, total_lines);
    }

    VPRINT(VERBOSE1, "%s%lu of %lu\n", "Number of Stored FQDNs: ", number_successful_fqdns, number_name_lookups);
    VPRINT(VERBOSE1, "%s%lu of %lu\n", "Number of Successful Retries: ", number_successful_retries, number_retries);

    close_dnshistory_db(&dnshistory_db_ptr);
    return (V_EXIT_OK);
}

/************************************************************************
 ************************************************************************
 *                              END                                     *
 ************************************************************************
 ************************************************************************/


/************************************************************************
 * add_address                                                          *
 *                                                                      *
 * Given an IP Address, do the lookup, and add the FQDN to our DNS DB   *
 ************************************************************************/
void
add_address(char *str_ipaddr)
{
    /* Logic:
     *  Convert from char to ip_addr
     *  Seen this IP address today?
     *    Yes:
     *      exit ok
     *    No:
     *      Lookup this IP Address (using char form!)
     *      New record?
     *        Yes:
     *          Initialize record and add to db
     *            date_set and date_last to same time
     *          Create new record
     *        No:
     *          Has the fqdn changed?
     *            Yes:
     *              Create a new record with date_set and date_last same time
     *            No:
     *              Update date_last to current day
     *              Update existing record
     */

    /**********************************************************************/
    dns_record_t *dnsrec;                       /* Current DNS Record to work with */
    pthread_t pthrd;                            /* Thread id - ignored */
    pthread_attr_t pthrd_attrs;                 /* Thread Attributes */
    int ret_thrd;                               /* Thread function results */
    int ret_pton;                               /* Results: inet_pton */
    struct dns_record_lists_t *list, *list_next;        /* list cleanup pointers */

    /**********************************************************************/

    VPRINT(VERBOSE5, "  INIT. Add Address; Do Lookup on: %s\n", str_ipaddr);

    dnsrec = XMALLOC(dns_record_t, 1);
    pthread_mutex_lock(&mutex_malloc_dns_rec);
    malloc_dns_rec++;
    pthread_mutex_unlock(&mutex_malloc_dns_rec);
    dnsrec->list = NULL;                        /* Probably unnecessary, but can't hurt */

    /* Convert from char string to in_addr_t */
    ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
    if (ret_pton == 0) {
        /* XXX What happens if already converted??? */
        ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
        return;
    }

    /* Set the stack size for threads */
    pthread_attr_init(&pthrd_attrs);
    ret_thrd = pthread_attr_setstacksize(&pthrd_attrs, THREAD_STACK_SIZE);
    if (ret_thrd != 0) {
        ERRVPRINT(VERBOSE0, msg_E_thread_stack_resize, ret_thrd);
    }

    if (!address_exists(dnsrec)) {
        /* Is a new record. So, create threads with reduced stack and fire off */
        VPRINT(VERBOSE5, "    New Record%s", "\n");
        /* First lets store a copy of this record, so we don't do repeated name lookups */
        dnsrec->date_last = current_day;
        number_added_addresses++;
        store_dns_records(dnsrec);
        increase_thread_counter();
        ret_thrd = pthread_create(&pthrd, &pthrd_attrs, add_new_address, (void *) dnsrec);
        if (ret_thrd != 0) {
            ERRVPRINT(VERBOSE0, msg_E_thread_creation, ret_thrd);
        }
    } else {
        /* We have seen this record before */
        VPRINT(VERBOSE5, "    Old Record: %lu --> %lu\n", dnsrec->date_last, current_day);
        if (dnsrec->date_last < (current_day - g_dns_timeout)) {
            /* But not in our timeout period - so need to update with a new record! */
            VPRINT(VERBOSE5, "     Outside timeout period. New Lookup.%s", "\n");
            /* First store an updated "date_last".
             * This way later requests will ignore this record if the name lookup takes a while */
            dnsrec->date_last = current_day;
            number_added_addresses++;
            store_dns_records(dnsrec);
            increase_thread_counter();
            ret_thrd = pthread_create(&pthrd, &pthrd_attrs, add_old_address, (void *) dnsrec);
            if (ret_thrd != 0) {
                ERRVPRINT(VERBOSE0, msg_E_thread_creation, ret_thrd);
            }
        } else {
            /* Can ignore this record, deallocate it and all it's links */
            list = dnsrec->list;
            VPRINT(VERBOSE5, "    Inside timeout period. Free Record(s).%s", "\n");
            while (list != NULL) {
                VPRINT(VERBOSE5, "      Freeing List Entry: %s\n", list->fqdn);
                list_next = list->next;
                XFREE(list);
                list = list_next;
                pthread_mutex_lock(&mutex_malloc_dns_list);
                malloc_dns_list--;
                pthread_mutex_unlock(&mutex_malloc_dns_list);
            }
            XFREE(dnsrec);
            pthread_mutex_lock(&mutex_malloc_dns_rec);
            malloc_dns_rec--;
            pthread_mutex_unlock(&mutex_malloc_dns_rec);
        }
    }
}


/************************************************************************
 * retrieve_address                                                     *
 *                                                                      *
 * Given an IP Address, do the lookup,                                  *
 *  and return the FQDN from the DNS DB                                 *
 *                                                                      *
 * str_fqdn[0] is set to '\0' on a "no valid answer" answer.            *
 ************************************************************************/
void
retrieve_address(char *str_ipaddr, char *str_fqdn, char *str_time)
{
    /**********************************************************************/
    dns_record_t *dnsrec;                       /* Current DNS Record to work with */
    struct dns_record_lists_t *list, *list_next;        /* list cleanup/traversing pointers */
    int ret_pton;                               /* Results: inet_pton */
    struct tm time_rec;                         /* Gotta convert that string'ed time into a timerec first */
    time_t time_logentry = 0;                   /* The current log time */
    int ret_addrexists = 0;
    time_t temp_time_squid;                     /* For pulling in squid times */

    /**********************************************************************/

    VPRINT(VERBOSE5, "  INIT. Retrieve Address; Looking for: %s\n", str_ipaddr);

    dnsrec = XMALLOC(dns_record_t, 1);
    dnsrec->list = NULL;                        /* Probably unnecessary, but can't hurt */

    /* Convert from char string to in_addr_t */
    ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
    if (ret_pton == 0) {
        /* XXX What happens if already converted??? */
        ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
        str_fqdn[0] = '\0';
    } else {

        ret_addrexists = address_exists(dnsrec);
        if ((ret_addrexists = 0) || (dnsrec->list == NULL)) {
            /* No answer to give */
            str_fqdn[0] = '\0';
        } else {
            /* Find and return the correct answer by timestamp */
            /******************
             * Use this entry if no older entry
             * time_logentry newer or same as date_set use this entry
             * time_logentry older than date_set, try next older;
             ******************/
            list = dnsrec->list;

            if (list->next != NULL) {
                memset(&time_rec, 0, sizeof(time_rec)); /* reset time_rec - failing to do so causes time comparison problems. Yuk */
                switch (g_log_type) {
                case LOG_CLF:
                    strptime(str_time, DATE_TIME_FORMAT, &time_rec);
                    break;
                case LOG_FTP:
                    strptime(str_time, DATE_TIME_XFERLOG_FORMAT, &time_rec);
                    break;
               case LOG_IPTABLES:
                    strptime(str_time, DATE_TIME_IPTABLES_FORMAT, &time_rec);
                    break;
                case LOG_SQUID:
                    temp_time_squid = strtoul(str_time, NULL, 10);
                    localtime_r(&temp_time_squid, &time_rec);
                    break;
                }

                time_logentry = mktime(&time_rec);
                VPRINT(VERBOSE5, "    **Date/Time: %s -> %ld\n", str_time, time_logentry);
                VPRINT(VERBOSE5, "    **Date/Time: DB START: %ld -> %ld\n", time_logentry, dnsrec->date_last);

                while (list != NULL) {
                    VPRINT(VERBOSE5, "    **Date/Time: DB List: %ld -> %ld\n", time_logentry, list->date_set);
                    if ((list->next == NULL) || (difftime(time_logentry, list->date_set) >= 0)) {
                        strncpy(str_fqdn, list->fqdn, NI_MAXHOST);
                    }
                    list = list->next;
                }
            } else {
                /* We only have one entry, so no need to do a time compare */
                strncpy(str_fqdn, list->fqdn, NI_MAXHOST);
            }
        }
    }

    /* Can ignore this record, deallocate it and all it's links */
    list = dnsrec->list;
    VPRINT(VERBOSE5, "    Inside timeout period. Free Record(s).%s", "\n");
    while (list != NULL) {
        VPRINT(VERBOSE5, "      Freeing List Entry: %s\n", list->fqdn);
        list_next = list->next;
        XFREE(list);
        list = list_next;
    }
    XFREE(dnsrec);

    return;
}


/************************************************************************
 * add_recombined_address                                               *
 *                                                                      *
 * Given an IP Address and FQDN , add to our DNS DB                     *
 ************************************************************************/
void
add_recombined_address(char *str_ipaddr, char *str_fqdn, char *str_time)
{
    /* Logic:
     */

    /**********************************************************************/
    dns_record_t *dnsrec;                       /* Current DNS Record to work with */
    int ret_pton;                               /* Results: inet_pton */
    struct dns_record_lists_t *list, *list_next;        /* list cleanup pointers */
    struct dns_record_lists_t dnsrec_list;      /* Single "list" entry for this record */
    struct tm time_rec;                         /* Gotta convert that string'ed time into a timerec first */
    time_t time_logentry = 0;                   /* The current log time */

    /**********************************************************************/

    VPRINT(VERBOSE3, "  **INIT: Add Recombined Address; Given: %s  Add: %s\n", str_ipaddr, str_fqdn);

    dnsrec = XMALLOC(dns_record_t, 1);
    dnsrec->list = NULL;                        /* Probably unnecessary, but can't hurt */

    /* Convert from char string to in_addr_t */
    ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec->ipaddress);
    if (ret_pton == 0) {
        /* XXX What happens if already converted??? */
        ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
        return;
    }

    /* Convert the Log Date/Time to time_t */
    memset(&time_rec, 0, sizeof(time_rec));     /* reset time_rec - failing to do so causes time comparison problems. Yuk */
    strptime(str_time, DATE_TIME_FORMAT, &time_rec);
    time_logentry = mktime(&time_rec);
    VPRINT(VERBOSE5, "    ARA: Date/Time: %s -> %ld\n", str_time, time_logentry);

    /************************************************
     * New Record?
     ************************************************/
    if (!address_exists(dnsrec)) {
        /* This is a new record. */
        VPRINT(VERBOSE4, "    ARA: New Record%s", "\n");

        memset(&dnsrec_list, 0, sizeof(dnsrec_list));
        dnsrec->list = &dnsrec_list;

        strncpy(dnsrec->list->fqdn, str_fqdn, NI_MAXHOST);
        dnsrec->date_last = time_logentry;
        dnsrec->list->date_set = time_logentry;
        dnsrec->list->next = NULL;

        /* Store this newly updated new record */
        VPRINT(VERBOSE2, "  ARA: Storing New: %15s %s\n", str_ipaddr, str_fqdn);
        number_added_addresses++;
        store_dns_records(dnsrec);
        XFREE(dnsrec);
    } else {
        /************************************************
         * Not New Record!
         ************************************************/
        VPRINT(VERBOSE5, "    ARA: Old Record: %lu --> %lu\n", dnsrec->date_last, time_logentry);
        if (dnsrec->date_last < (time_logentry - g_dns_timeout)) {
            /* But not in our timeout period - so need to update with a new record! */
            VPRINT(VERBOSE5, "      ARA: Outside timeout period. Update Entry.%s", "\n");
            /* First store an updated "date_last".
             * This way later requests will ignore this record if the name lookup takes a while */
            dnsrec->date_last = time_logentry;
            if (strncmp(dnsrec->list->fqdn, str_fqdn, NI_MAXHOST) != 0) {
                /* This is a new and updated FQDN, insert */
                VPRINT(VERBOSE4, "      ARA: Add Recombined Address: Changed FQDN: %s --> %s\n", dnsrec->list->fqdn, str_fqdn);
                list_next = XMALLOC(struct dns_record_lists_t, 1);

                list_next->date_set = time_logentry;
                strncpy(list_next->fqdn, str_fqdn, NI_MAXHOST);
                list_next->next = dnsrec->list;
                dnsrec->list = list_next;
                VPRINT(VERBOSE2, "  ARA: Storing Update: %15s %s\n", str_ipaddr, str_fqdn);
            }
            number_added_addresses++;
            store_dns_records(dnsrec);          /* We only need to store an updated value, we've already stored a new time */
        }
        /* Deallocate it and all it's links */
        list = dnsrec->list;
        VPRINT(VERBOSE5, "      ARA: Inside timeout period. Free Record(s).%s", "\n");
        while (list != NULL) {
            VPRINT(VERBOSE5, "        ARA: Freeing List Entry: %s\n", list->fqdn);
            list_next = list->next;
            XFREE(list);
            list = list_next;
        }
        XFREE(dnsrec);
    }
}


/************************************************************************
 * name_lookup                                                          *
 *                                                                      *
 * Do the name lookup and error handling                                *
 *   Requires a string[NI_MAXHOST] to dump the results into             *
 *    -> host_name                                                      *
 ************************************************************************/
int
name_lookup(struct in_addr ipaddr, char *host_name)
{
    /**********************************************************************/
    int ret_gni = 0;                            /* getnameinfo Return value */
    struct sockaddr_in sa;                      /* Structure to send to getnameinfo */

    sa.sin_family = AF_INET;                    /* IPv4 lookup */
    sa.sin_port = 0;                            /* Don't care about the port */
    sa.sin_addr.s_addr = ipaddr.s_addr;         /* Set the IP address to lookup */
    int retries = g_dns_lookups;
    int err;                                    /* Error Checking */
    bool doing_retry = 0;                       /* Flag to help count number of retries */

    /**********************************************************************/

    err = pthread_mutex_lock(&mutex_other_counters);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    number_name_lookups++;                      /* Increase number of lookups counter */
    err = pthread_mutex_unlock(&mutex_other_counters);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);

    /* Do the lookup */
    while (retries > 0) {
        ret_gni = getnameinfo((struct sockaddr *) &sa, sizeof(sa), host_name, NI_MAXHOST, NULL, 0, NI_NAMEREQD);

        /* Error/Success checking */
        if (ret_gni != 0) {
            if ((ret_gni == EAI_AGAIN) || (ret_gni == EAI_NONAME)) {
                VPRINT(VERBOSE3, "NameLookup: TryAgain: %s\n", host_name);
                retries--;
                if (retries > 0) {

                    err = pthread_mutex_lock(&mutex_other_counters);
                    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
                    number_retries++;
                    err = pthread_mutex_unlock(&mutex_other_counters);
                    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);

                    if (doing_retry == false) {
                        doing_retry = true;
                    }
                    sleep(g_dns_retry_delay);
                }
            } else {
                name_lookup_errors(ret_gni);
                break;
            }
        } else {
            VPRINT(VERBOSE2, "NameLookup: Success: %s\n", host_name);
            break;
        }
    }

    check_n_fix_fqdn(host_name);

    if (ret_gni == 0) {
        err = pthread_mutex_lock(&mutex_other_counters);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
        number_successful_fqdns++;
        err = pthread_mutex_unlock(&mutex_other_counters);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
    }

    if ((ret_gni == 0) && (doing_retry == true)) {
        err = pthread_mutex_lock(&mutex_other_counters);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
        number_successful_retries++;
        err = pthread_mutex_unlock(&mutex_other_counters);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
    }
    /* Even tho we have a technical failure, we want to return a success for ipaddy == ipaddy lookups */
    if (ret_gni == EAI_NONAME) {
        ret_gni = 0;
    }
    return (ret_gni);
}


/************************************************************************
 * increase_thread_counter                                              *
 *                                                                      *
 * Do as the name says. Lock; increase; Unlock; exit. Duh.              *
 ************************************************************************/
void
increase_thread_counter()
{
    /**********************************************************************/
    int err;                                    /* Error Checking */

    /**********************************************************************/

    /* Increase the count of the number of threads running */
    err = pthread_mutex_lock(&mutex_thread_count);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    /* thread_count is a global */
    thread_count++;
    if (thread_count > thread_count_max) {
        thread_count_max = thread_count;
    }
    VPRINT(VERBOSE4, "THRD START: Thread count: %d\n", thread_count);
    err = pthread_mutex_unlock(&mutex_thread_count);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
}


/************************************************************************
 * decrease_thread_counter                                              *
 *                                                                      *
 * Do as the name says. Lock; decrease; Unlock; Signal; exit.           *
 * We do a thread cond signal as well - just to alert for file end      *
 *   processing.                                                        *
 ************************************************************************/
void
decrease_thread_counter()
{
    /**********************************************************************/
    int err;                                    /* Error Checking */

    /**********************************************************************/

    /* Decrease # of thread's
     * Signal that this thread has finished - used for end of file processing */
    err = pthread_mutex_lock(&mutex_thread_count);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    /* thread_count is a global */
    thread_count--;
    VPRINT(VERBOSE4, "THRD EXIT: Thread count: %d\n", thread_count);
    pthread_cond_signal(&cond_thread_count);
    err = pthread_mutex_unlock(&mutex_thread_count);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
}


/************************************************************************
 * add_new_address                                                      *
 *                                                                      *
 * Do the name lookup and error handling                                *
 *   Requires a string[NI_MAXHOST] to dump the results into             *
 * This is a threaded function                                          *
 ************************************************************************/
void *
add_new_address(void *arg)
        /* ip_address : is of type "struct in_addr" */
{
    /**********************************************************************/
    int ret_gni;                                /* getnameinfo Return value */
    dns_record_t *dnsrec = arg;                 /* Current DNS Record to work with */
    struct dns_record_lists_t dnsrec_list;      /* Single "list" entry for this record */
    int err;                                    /* Error Checking */

    /**********************************************************************/

    VPRINT(VERBOSE5, "ADD_NEW_ADDR: START!:%s", "\n");

    /* Initialize the records */
    memset(&dnsrec_list, 0, sizeof(dnsrec_list));       /* Not strictly necessary, but just in case */
    dnsrec->list = &dnsrec_list;

    /* Do the lookup */
    ret_gni = name_lookup(dnsrec->ipaddress, dnsrec->list->fqdn);

    /* Error/Success checking */
    if (ret_gni != 0) {
        name_lookup_errors(ret_gni);
    } else {
        dnsrec->date_last = current_day;
        dnsrec->list->date_set = current_day;
        dnsrec->list->next = NULL;
    }

    /* Store this newly updated new record */
    store_dns_records(dnsrec);
    XFREE(dnsrec);

    err = pthread_mutex_lock(&mutex_malloc_dns_rec);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    malloc_dns_rec--;
    err = pthread_mutex_unlock(&mutex_malloc_dns_rec);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);

    decrease_thread_counter();
    /* Detach this thread - returning memory etc
     * Exit. */
    pthread_detach(pthread_self());
    pthread_exit(0);
}


/************************************************************************
 * add_old_address                                                      *
 *                                                                      *
 * Do the name lookup for an old record                                 *
 * Wants a dns_record_t argument passed in.                             *
 *  Assumes this has been malloc'd and will then free it                *
 * This is a threaded function                                          *
 ************************************************************************/
void *
add_old_address(void *arg)
        /* dns_record is a pointer to struct dns_record_t */
{
    /**********************************************************************/
    int ret_gni;                                /* getnameinfo Return value */
    dns_record_t *dnsrec = arg;                 /* Current DNS Record to work with */
    struct dns_record_lists_t dnsrec_list;      /* Single "list" entry for this record */
    struct dns_record_lists_t *list, *list_next;        /* list cleanup pointers */
    int err;                                    /* Error Checking */

    /**********************************************************************/

    /* increase_thread_counter (); */

    /* Initialize the records */
    memset(&dnsrec_list, 0, sizeof(dnsrec_list));       /* Not strictly necessary, but just in case */

    /* Do the lookup */
    ret_gni = name_lookup(dnsrec->ipaddress, dnsrec_list.fqdn);

    /* Error/Success checking */
    if (ret_gni != 0) {
        name_lookup_errors(ret_gni);
    } else {
        /* Check the most recent record for this IP Address */
        if (strncmp(dnsrec->list->fqdn, dnsrec_list.fqdn, NI_MAXHOST) != 0) {
            /* This is a new and updated FQDN, insert */
            VPRINT(VERBOSE2, "ADD_OLD_ADDR: Changed FQDN: %s --> %s\n", dnsrec->list->fqdn, dnsrec_list.fqdn);
            dnsrec->date_last = current_day;
            dnsrec_list.date_set = current_day;
            dnsrec_list.next = dnsrec->list;
            dnsrec->list = &dnsrec_list;
            store_dns_records(dnsrec);          /* We only need to store an updated value, we've already stored a new time */
        }
    }

    /* Final tidy up - free the list of FQDN's */
    list = dnsrec->list;
    if (dnsrec->list == &dnsrec_list) {         /* Don't try and free our local variable dnsrec_list */
        list = dnsrec->list->next;
    }
    while (list != NULL) {
        VPRINT(VERBOSE5, "ADD_OLD_ADDR: Freeing List Entry: %s\n", list->fqdn);
        list_next = list->next;
        XFREE(list);
        list = list_next;

        err = pthread_mutex_lock(&mutex_malloc_dns_list);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
        malloc_dns_list--;
        err = pthread_mutex_unlock(&mutex_malloc_dns_list);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
    }
    XFREE(dnsrec);
    err = pthread_mutex_lock(&mutex_malloc_dns_rec);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    malloc_dns_rec--;
    err = pthread_mutex_unlock(&mutex_malloc_dns_rec);
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
    decrease_thread_counter();

    VPRINT(VERBOSE5, "ADD_OLD_ADDR: EXITING!: %s\n", dnsrec_list.fqdn);
    /* Detach this thread - returning memory etc
     * Exit. */
    pthread_detach(pthread_self());
    pthread_exit(0);
}


/************************************************************************
 * name_lookup_errors                                                   *
 *                                                                      *
 * Deal with Name Lookup errors (getnameinfo)                           *
 ************************************************************************/
void
name_lookup_errors(int error_code)
{
    if (g_verbosity >= VERBOSE1) {
        switch (error_code) {
        case EAI_AGAIN:
            ERRVPRINT(VERBOSE0, "NameLookup: %s\n", msg_namelookup_try_agin);
            break;
        case EAI_FAIL:
            ERRVPRINT(VERBOSE0, "NameLookup: %s\n", "Fatal Error");
            break;
        default:
            ERRVPRINT(VERBOSE0, "NameLookup: %d %s\n", error_code, msg_namelookup_default);
            break;
        }
    }
}


/************************************************************************
 * drop_long_lines                                                      *
 *                                                                      *
 * Strip Log Lines that are too big                                     *
 ************************************************************************/
void
drop_long_lines(gzFile * file_input, char *buffer_ptr, buffer_position * buf_posn)
{
    /**********************************************************************/
    char *fgets_rtn = NULL;

    /**********************************************************************/

    /* Strip Log Lines that are too big */
    buffer_ptr[BUFCUTOFF] = '\0';

    ERRVPRINT(VERBOSE1, "#%lu  %s :%s...\n", total_lines, msg_W_line_too_big, buffer_ptr);
    while (1) {
        if (file_input == NULL) {
            fgets_rtn = fgets(buffer_ptr, BUFSIZE, stdin);
        } else {
            fgets_rtn = get_log_line(buffer_ptr, BUFSIZE, file_input, buf_posn);
            /* ERRVPRINT (VERBOSE2, "  %s\n", buffer_ptr); */
        }
        if (strpbrk("\n", buffer_ptr) != NULL) {
            break;
        }
    }
}


/************************************************************************
 * get_log_line                                                         *
 *                                                                      *
 * Get the next log line (\n terminated)                                *
 * Is designed to deal with gz'ed logs, but will also do plain text     *
 * The buffer_position structure is for tracking the raw de-gziped      *
 *   buffer and where we are up to within it.                           *
 ************************************************************************/
char *
get_log_line(char *buf, int size, gzFile * file_ptr, buffer_position * buf_posn)
{
    /* Faster version of gzgets.
     * Original code and algorithim taken from mergelog-4.5
     *          by Bertrand Demiddelaer
     *
     * Has been quite hacked around to fix some issues in the re-implementation.
     *   Lack of a terminating '\n' in the original being the biggest problem.
     */

    /**********************************************************************/
    char *out_copy = buf;
    int bytes_returned;
    int size_ctr = 0;

    /**********************************************************************/

    VPRINT(VERBOSE5, "\n##GLL: Start: %-10p Cur: %-10p End: %-10p\n", buf_posn->decomp_buf, buf_posn->current_pos_ptr, buf_posn->end_decompbuf_ptr);
    while (1) {
        if (buf_posn->current_pos_ptr > buf_posn->end_decompbuf_ptr) {
            bytes_returned = gzread(file_ptr, buf_posn->decomp_buf, DECOMP_BUFSIZE);
            if (bytes_returned <= 0) {
                return (NULL);
            }
            buf_posn->end_decompbuf_ptr = buf_posn->decomp_buf + ((bytes_returned - 1) * sizeof(char));
            buf_posn->current_pos_ptr = buf_posn->decomp_buf;
        }

        if (size_ctr >= size - 1) {
            /* This buffer is now full - we've exceeded line size */
            buf[size - 1] = '\0';
            return (buf);
        } else {
            /* Copy each char from the ungzipped buffer to the line request buffer */
            *out_copy = *buf_posn->current_pos_ptr;
            if (*buf_posn->current_pos_ptr == '\n') {
                out_copy++;
                *out_copy = '\0';
                buf_posn->current_pos_ptr++;    /* We will be leaving shortly, so increment to next character */
                return (buf);
            }
        }
        size_ctr++;
        out_copy++;
        buf_posn->current_pos_ptr++;
    }
}


/************************************************************************
 * close_exit                                                           *
 *                                                                      *
 * Close the open Database and Exit                                     *
 *                                                                      *
 * Arguments:                                                           *
 * FILE * file_input            The input File. If NULL, use STDIN.     *
 * int exit_code                The exit code to exit with.             *
 *                                Should be defined in error.h          *
 ************************************************************************/
void
close_exit(int exit_code)
{
    close_dnshistory_db(&dnshistory_db_ptr);
    exit(exit_code);
}


/************************************************************************
 * address_exists                                                       *
 *                                                                      *
 * Have we seen this address?                                           *
 * If yes, fill in remaining data fields and return 1                   *
 * If no, return 0                                                      *
 ************************************************************************/
int
address_exists(dns_record_t * dns_rec_ptr)
{
    /**********************************************************************/
    int rtn = 0;                                /* Return value */
    DBT dbt_key, dbt_data;                      /* Key and Data Pointers for results from BDB */
    int db_rtn;                                 /* return value from DBD Get */

    struct dns_record_lists_t *new_dns_rec = NULL;
    struct dns_record_lists_t *tail_dns_rec = NULL;
    void *idx_ptr = NULL;                       /* Counter. Where are we up to in the new memory block */

    int size = 0;                               /* Counter. How big is each data item for the memory block */
    int nbr_items = 0;                          /* Counter. How many data loops */
    int err;                                    /* Error Checking */

    char str_address[INET_ADDRSTRLEN];          /* Temp holder for displaying IP Address to lookup */

    /**********************************************************************/

    VPRINT(VERBOSE5, "    Address Exists?: %16s\n", inet_ntop(AF_INET, &dns_rec_ptr->ipaddress, str_address, INET_ADDRSTRLEN));

    memset(&dbt_key, 0, sizeof(dbt_key));
    memset(&dbt_data, 0, sizeof(dbt_data));

    dbt_key.data = (char *) &dns_rec_ptr->ipaddress.s_addr;
    dbt_key.size = sizeof(dns_rec_ptr->ipaddress.s_addr);

    err = pthread_mutex_lock(&mutex_db_access); /* Lock all DB accesses */
    ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
    db_rtn = dnshistory_db_ptr->get(dnshistory_db_ptr, NULL, &dbt_key, &dbt_data, 0);
    if (db_rtn == 0) {
        /* Yes! We have seen this address before. */
        idx_ptr = dbt_data.data;
        size = sizeof(dns_rec_ptr->date_last);
        memcpy(&dns_rec_ptr->date_last, idx_ptr, size);
        idx_ptr += size;

        size = sizeof(nbr_items);
        memcpy(&nbr_items, idx_ptr, size);
        idx_ptr += size;

        while (nbr_items > 0) {
            /* Build up the structure of old addresses */
            new_dns_rec = XMALLOC(struct dns_record_lists_t, 1);

            err = pthread_mutex_lock(&mutex_malloc_dns_list);
            ERR_NONZERO(err, VERBOSE0, msg_E_thread_lock, err);
            malloc_dns_list++;
            err = pthread_mutex_unlock(&mutex_malloc_dns_list);
            ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);

            /* Add this new rec to the end of the list */
            if (dns_rec_ptr->list == NULL) {
                dns_rec_ptr->list = new_dns_rec;
            } else {
                tail_dns_rec->next = new_dns_rec;
            }
            tail_dns_rec = new_dns_rec;
            new_dns_rec->next = NULL;

            /* Store date_set for this fqdn */
            size = sizeof(new_dns_rec->date_set);
            memcpy(&new_dns_rec->date_set, idx_ptr, size);
            idx_ptr += size;

            /* Store the fqdn itself */
            size = (strlen((char *) idx_ptr) + 1) * sizeof(char);
            memcpy(new_dns_rec->fqdn, idx_ptr, size);
            check_n_fix_fqdn(new_dns_rec->fqdn);
            idx_ptr += size;

            VPRINT(VERBOSE5, "      Address Exists? Yes: %s\n", new_dns_rec->fqdn);
            nbr_items--;
        }
        rtn = 1;
        err = pthread_mutex_unlock(&mutex_db_access);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
    } else {
        err = pthread_mutex_unlock(&mutex_db_access);
        ERR_NONZERO(err, VERBOSE0, msg_E_thread_unlock, err);
        VPRINT(VERBOSE5, "      Address Exists?: No.%s", "\n");
        rtn = 0;
    }
    return (rtn);
}


/************************************************************************
 * store_dns_records                                                    *
 *                                                                      *
 * Store the dns records for this lookup into the hash DB.              *
 * Compact any strings down to minimum levels.                          *
 ************************************************************************/
void
store_dns_records(dns_record_t * dns_rec_ptr)
{
    /* dns_rec_ptr: is the dns structure, and links, to store */
    /**********************************************************************/
    void *data_ptr = NULL;                      /* pointer to the data store to store in the hash */
    void *idx_ptr = NULL;                       /* Counter. Where are we up to in the new memory block */
    struct dns_record_lists_t *list_ptr;        /* pointer to index thru the dns history */

    int data_size = 0;                          /* size of fixed data in the dns record */
    int key_size = 0;                           /* size of the cookie itself - key for the hash */
    int nbr_items = 0;                          /* Counter. How many data loops */
    int size = 0;                               /* Counter. How big is each data item for the memory block */

    char str_address[INET_ADDRSTRLEN];          /* VPRINT buffer only! */

    /**********************************************************************/

    VPRINT(VERBOSE4, "  **STORE: %16s\n", inet_ntop(AF_INET, &dns_rec_ptr->ipaddress, str_address, INET_ADDRSTRLEN));

    key_size = sizeof(dns_rec_ptr->ipaddress.s_addr);
    data_size = sizeof(dns_rec_ptr->date_last);
    list_ptr = dns_rec_ptr->list;
    while (list_ptr != NULL) {
        data_size += sizeof(list_ptr->date_set);
        data_size += (strlen(list_ptr->fqdn) + 1) * sizeof(char);
        list_ptr = list_ptr->next;
        nbr_items++;
    }
    data_size += sizeof(nbr_items);

    /* Grab the memory needed for all the dns rec's data */
    data_ptr = (void *) malloc(data_size);
    ERR_NULL_EXIT(data_ptr, V_EXIT_MEMORY_EXHAUSTION, msg_F_memory_alloc, "");
    memset(data_ptr, 0, data_size);             /* Not strictly necessary, but just in case */

    /* Store the most recent date for this dns record */
    idx_ptr = data_ptr;
    size = sizeof(dns_rec_ptr->date_last);
    memcpy(idx_ptr, &dns_rec_ptr->date_last, size);
    idx_ptr += size;
    /* Store the number of items in the dns history, for later extraction! */
    size = sizeof(nbr_items);
    memcpy(idx_ptr, &nbr_items, size);
    idx_ptr += size;

    list_ptr = dns_rec_ptr->list;
    while (list_ptr != NULL) {
        /* Date this record was first seen */
        size = sizeof(list_ptr->date_set);
        memcpy(idx_ptr, &list_ptr->date_set, size);
        idx_ptr += size;

        /* FQDN storage */
        VPRINT(VERBOSE5, "    STO: fqdn: %s\n", list_ptr->fqdn);
        size = (strlen(list_ptr->fqdn) + 1) * sizeof(char);
        memcpy(idx_ptr, &list_ptr->fqdn, size);
        idx_ptr += size;

        list_ptr = list_ptr->next;
    }

    add_record(&dnshistory_db_ptr, &dns_rec_ptr->ipaddress.s_addr, key_size, data_ptr, data_size);
    XFREE(data_ptr);                            /* release the memory grabbed */
}


/************************************************************************
 * display_record                                                       *
 *                                                                      *
 * Given a BDB Data Key and Data Value pair, display the stored         *
 *   address information/history                                        *
 ************************************************************************/
void
display_record(DBT * dbt_key, DBT * dbt_data, bool realdate)
{
    dns_record_t dnsrec;                        /* Current DNS Record to work with */
    struct dns_record_lists_t dnslist_rec;      /* Current DNS Data to work with */

    void *idx_ptr = NULL;                       /* Counter. Where are we up to in the new memory block */
    int size = 0;                               /* Counter. How big is each data item for the memory block */
    int nbr_items = 0;                          /* Counter. How many data loops */
    char str_address[INET_ADDRSTRLEN];          /* Print buffer for converting stored IPAddresses to Normal w.x.y.z */

    struct tm *date_last_full;

    VPRINT(VERBOSE5, "  Display Record!%s", "\n");

    memset(&dnsrec, 0, sizeof(dnsrec));
    idx_ptr = dbt_data->data;
    size = sizeof(dnsrec.date_last);
    memcpy(&dnsrec.date_last, idx_ptr, size);
    idx_ptr += size;

    size = sizeof(nbr_items);
    memcpy(&nbr_items, idx_ptr, size);
    idx_ptr += size;

    /* Copy in the raw IP Address */
    memcpy(&dnsrec.ipaddress.s_addr, dbt_key->data, sizeof(dnsrec.ipaddress.s_addr));
    inet_ntop(AF_INET, &dnsrec.ipaddress, str_address, INET_ADDRSTRLEN);
    printf("%-s", str_address);

    /* Loop thru the various list items */
    while (nbr_items > 0) {
        memset(&dnslist_rec, 0, sizeof(dnsrec));
        /* Store date_set for this fqdn */
        size = sizeof(dnslist_rec.date_set);
        memcpy(&dnslist_rec.date_set, idx_ptr, size);
        idx_ptr += size;

        /* Store the fqdn itself */
        size = (strlen((char *) idx_ptr) + 1) * sizeof(char);
        memcpy(dnslist_rec.fqdn, idx_ptr, size);
        idx_ptr += size;

        printf("\t");

        if (realdate == true) {
            date_last_full = localtime(&dnslist_rec.date_set);
            printf("%4d-%02d-%02d:%02d:%02d:%02d,", date_last_full->tm_year + 1900, date_last_full->tm_mon + 1, date_last_full->tm_mday, date_last_full->tm_hour,
                   date_last_full->tm_min, date_last_full->tm_sec);
        } else {
            printf("%d,", (int) dnslist_rec.date_set);
        }

        check_n_fix_fqdn(dnslist_rec.fqdn);
        if (strlen(dnslist_rec.fqdn) > 0) {
            printf("%s", dnslist_rec.fqdn);
        } else {
            printf("%s", STR_NONAME);
//            printf ("%s", "nullname");
        }

        memset(&str_address, 0, sizeof(str_address));

        nbr_items--;
    }
    printf("\n");

}


/************************************************************************
 * dump_dns_historydb                                                   *
 *                                                                      *
 * Given a Database pointer, dump the DNS History values to stdout      *
 ************************************************************************/
void
dump_dns_historydb(DB ** db_ptr)
{
    /**********************************************************************/
    DBC *dbcurs_ptr = NULL;                     /* DB Pointer for cursor'ing */
    DBT dbt_key, dbt_data;                      /* Key and Data Pointers for results from BDB */
    int db_rtn;                                 /* return value from DBD Get */

    /**********************************************************************/

    VPRINT(VERBOSE5, "  History Dump!%s", "\n");
    /* Acquire a cursor for the database. */
    db_rtn = (*db_ptr)->cursor(*db_ptr, NULL, &dbcurs_ptr, 0);
    if (db_rtn != 0) {
        (*db_ptr)->err(*db_ptr, db_rtn, "DB->cursor");
        ERRVPRINT(VERBOSE0, msg_F_db_cursor, "");
        close_exit(V_EXIT_DB_CURSOR);
    }
    VPRINT(VERBOSE5, "dbcurs_ptr == %p", dbcurs_ptr);

    memset(&dbt_key, 0, sizeof(dbt_key));
    memset(&dbt_data, 0, sizeof(dbt_data));

    while ((db_rtn = dbcurs_ptr->c_get(dbcurs_ptr, &dbt_key, &dbt_data, DB_NEXT)) == 0) {
        display_record(&dbt_key, &dbt_data, false);
    }
    db_rtn = dbcurs_ptr->c_close(dbcurs_ptr);
}


/************************************************************************
 * import_dns_historydb                                                 *
 *                                                                      *
 * Given a Database pointer, import the DNS History values to this BDB  *
 ************************************************************************/
void
import_dns_historydb(void)
{
    FILE *import_file;
    char *fgets_rtn = NULL;
    char buffer[BUFSIZE];

    char str_address[INET_ADDRSTRLEN];          /* Print buffer for converting stored IPAddresses to Normal w.x.y.z */

    dns_record_t dnsrec;                        /* Current DNS Record to work with */

    struct dns_record_lists_t *last_list_ptr;   /* pointer to index thru the dns history */
    struct dns_record_lists_t *new_dns_rec = NULL;      /* New list pointer to malloc */
    struct dns_record_lists_t *list, *list_next;        /* list cleanup pointers */

    int rtn_sscanf = 0;                         /* sscanf return value, for later checking */
    bool flag_is_bad_line;

    /**********************************************************************/
    VPRINT(VERBOSE1, "DNS History Import!%s", "\n");

    import_file = fopen(g_import_filename, "r");
    if (import_file == NULL) {
        ERRVPRINT(VERBOSE0, msg_F_file_open, g_import_filename);
        exit(1);
    }

    while (1) {
        fgets_rtn = fgets(buffer, BUFSIZE, import_file);
        if (fgets_rtn == NULL) {
            /* Exit Main Loop at end of input */
            break;
        }
        total_lines++;
        VPRINT(VERBOSE5, "Line: %lu  Importing: %s", total_lines, buffer);
        flag_is_bad_line = false;

        /* check for log line too long */
        if (strpbrk(buffer, "\n") == NULL) {
            ERRVPRINT(VERBOSE0, msg_W_import_line_too_long, total_lines);
            while (1) {
                /* Loop over import till we reach the end of this line, then break and get the next line */
                fgets_rtn = fgets(buffer, BUFSIZE, import_file);
                if (strpbrk("\n", buffer) != NULL) {
                    break;
                }
            }
            bad_lines++;
            continue;
        }

        char *buf_ptr1 = NULL;
        char *buf_ptr2 = NULL;

        memset(&dnsrec, 0, sizeof(dnsrec));

        rtn_sscanf = sscanf(buffer, "%15s ", (char *) &str_address);
        if (rtn_sscanf < 1) {
            ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
            bad_lines++;
            continue;
        }

        inet_pton(AF_INET, str_address, &dnsrec.ipaddress);
        dnsrec.list = NULL;
        last_list_ptr = NULL;

        buf_ptr1 = strpbrk(buffer, "\t");       /* First tab */
        if (buf_ptr1 == NULL) {
            ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
            bad_lines++;
            continue;
        }
        buf_ptr2 = buf_ptr1;

        /* Loop over each tab separator */
        while (buf_ptr2 != NULL) {
            new_dns_rec = XMALLOC(struct dns_record_lists_t, 1);

            if (dnsrec.list == NULL) {
                dnsrec.list = new_dns_rec;
            }

            rtn_sscanf = sscanf(buf_ptr2, " %lu,%1024s ", (unsigned long *) &new_dns_rec->date_set, (char *) &new_dns_rec->fqdn);
            VPRINT(VERBOSE2, "    Scan: %lu --> %s\n", (unsigned long) new_dns_rec->date_set, new_dns_rec->fqdn);
            if (rtn_sscanf < 2) {
                ERRVPRINT(VERBOSE0, msg_W_import_line_failure, total_lines);
                bad_lines++;
                flag_is_bad_line = true;
                break;
            }

            if (dnsrec.date_last == 0) {
                dnsrec.date_last = new_dns_rec->date_set;
            }
            if (last_list_ptr != NULL) {
                last_list_ptr->next = new_dns_rec;
            }
            last_list_ptr = new_dns_rec;
            new_dns_rec->next = NULL;

            /* See if we've grabbed a 'NONAME' value, replace with a null if so
             * Don't want 'NONAME' as the FQDN... */
            if (strncmp(new_dns_rec->fqdn, STR_NONAME, strlen(new_dns_rec->fqdn)) == 0) {
                new_dns_rec->fqdn[0] = '\0';
            } else {
                check_n_fix_fqdn(new_dns_rec->fqdn);
            }

            /* Find next tabstop */
            buf_ptr1 = buf_ptr2;
            buf_ptr2 = strpbrk(buf_ptr1 + 1, "\t");
        }

        /* Don't store if we've got a funky internal/additional sscanf */
        if (flag_is_bad_line != true) {
            store_dns_records(&dnsrec);
            number_successful_fqdns++;
        }

        /* Final tidy up - free the list of FQDN's */
        if (dnsrec.list != NULL) {
            list = dnsrec.list;
        } else {
            /* Should *never* be here - implies no valid line data - bad! */
            list = NULL;
        }
        while (list != NULL) {
            ERRVPRINT(VERBOSE3, "  Freeing List Entry: %s\n", list->fqdn);
            list_next = list->next;
            XFREE(list);
            list = list_next;

        }

    }                                           /* while (1) */

    VPRINT(VERBOSE0, "Successfully Imported %lu Records\n", number_successful_fqdns);
    if (bad_lines > 0) {
        VPRINT(VERBOSE0, "%s%lu of %lu\n", msg_I_number_bad_lines, bad_lines, total_lines);
    }

}


/************************************************************************
 * showhistory                                                          *
 *                                                                      *
 * Given a Database pointer, dump the DNS History values to stdout      *
 ************************************************************************/
void
showhistory(DB ** db_ptr, int argc, char *argv[])
{
    /**********************************************************************/
    char *str_ipaddr;                           /* The current IP address from cmd line */
    int ret_pton;                               /* Results: inet_pton */

    DBT dbt_key, dbt_data;                      /* Key and Data Pointers for results from BDB */
    int db_rtn;                                 /* return value from DBD Get */

    dns_record_t dnsrec;                        /* Current DNS Record to work with */
    int arg_ctr = optind;

    /**********************************************************************/

    VPRINT(VERBOSE5, "  History Lookup!%s", "\n");
    VPRINT(VERBOSE5, "    Doing  %d\n", argc);

    while (arg_ctr < argc) {
        str_ipaddr = argv[arg_ctr];
        arg_ctr++;
        VPRINT(VERBOSE5, "    IP: %s\n", str_ipaddr);
        /* Convert from char string to in_addr_t */
        ret_pton = inet_pton(AF_INET, str_ipaddr, &dnsrec.ipaddress);
        if (ret_pton == 0) {
            ERRVPRINT(VERBOSE0, msg_E_ip_conversion, ret_pton, str_ipaddr);
            continue;
        }
        memset(&dbt_key, 0, sizeof(dbt_key));
        memset(&dbt_data, 0, sizeof(dbt_data));

        dbt_key.data = (char *) &dnsrec.ipaddress.s_addr;
        dbt_key.size = sizeof(dnsrec.ipaddress.s_addr);

        db_rtn = (*db_ptr)->get(*db_ptr, NULL, &dbt_key, &dbt_data, 0);
        if (db_rtn == 0) {
            display_record(&dbt_key, &dbt_data, true);
        } else {
            printf("%-s\t%s\n", str_ipaddr, "ADDRESS NOT FOUND");
        }
    }                                           /* while */
}


/************************************************************************
 * check_n_fix_fqdn                                                     *
 *                                                                      *
 * Given a FQDN string, replace all non printable ASCII chars with      *
 * underscores.                                                         *
 ************************************************************************/
void
check_n_fix_fqdn(char *fqdn)
{
    unsigned int length, i;

    VPRINT(VERBOSE4, "  Check'N Fix: %s --> ", fqdn);
    length = strlen(fqdn);
    for (i = 0; i < length; i++) {
        if ((fqdn[i] < 33) || (fqdn[i] > 126)) {
            fqdn[i] = '_';
        }
    }
    VPRINT(VERBOSE4, "%s\n", fqdn);
}


/************************************************************************
 * identify_log_format                                                  *
 *                                                                      *
 * Attempt to identify the type of log format we've been given.         *
 * Returns the LOG_type as defined in dnshistory.h                      *
 * returns -1 if unknown.                                               *
 *                                                                      *
 * Requires a line of the log to attempt to process                     *
 ************************************************************************/
int
identify_log_format(char *buffer)
{
    int ovector[OVECCOUNT];                     /* RE substring offsets array */
    int rc;                                     /* RE Check return value */
    int buffer_length;


    buffer_length = (int) strlen(buffer);

    rc = pcre_exec(cmp_log_regexp_clf, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
    if (rc >= 0) {
        /* Matches against CLF */
        VPRINT(VERBOSE1, "Using CLF Log Format%s", "\n");
        return (LOG_CLF);
    }

    rc = pcre_exec(cmp_log_regexp_xferlog, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
    if (rc >= 0) {
        /* Matches against FTP/XFERLOG */
        VPRINT(VERBOSE1, "Using FTP/XFERLOG Log Format%s", "\n");
        return (LOG_FTP);
    }

    rc = pcre_exec(cmp_log_regexp_squid, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
    if (rc >= 0) {
        /* Matches against SQUID */
        VPRINT(VERBOSE1, "Using SQUID Log Format%s", "\n");
        return (LOG_SQUID);
    }

    rc = pcre_exec(cmp_log_regexp_iptables, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
    if (rc >= 0) {
        /* Matches against IPTABLES */
        VPRINT(VERBOSE1, "Using IPTABLES Log Format%s", "\n");
        return (LOG_IPTABLES);
    }

    rc = pcre_exec(cmp_log_regexp_syslog, NULL, buffer, buffer_length, 0, 0, ovector, OVECCOUNT);
    if (rc >= 0) {
        /* Matches against SYSLOG */
        VPRINT(VERBOSE1, "Using SYSLOG/IPTABLES Log Format%s", "\n");
        return (LOG_IPTABLES);
    }

    VPRINT(VERBOSE1, "Unrecognised Log Format%s", "\n");
    return (-1);                                /* Failed to match any, unknown format */
}


/************************************************************************
 * re_compile_all_regexs                                                *
 *                                                                      *
 * Does what the name says, in a single function we compile all         *
 *  possibly used Regular expressions.                                  *
 * Either forcibly exits on any failure, or happily finishes.           *
 * No values needed or returned.                                        *
 *                                                                      *
 * Assigns the RE's to the various globals:                             *
 *   cmp_log_regexp_*                                                   *
 ************************************************************************/
void
re_compile_all_regexes(void)
{
    char log_regexp_clf[MAX_RE_LENGTH] = PATTERN_CLF;
    char log_regexp_xferlog[MAX_RE_LENGTH] = PATTERN_XFERLOG;
    char log_regexp_squid[MAX_RE_LENGTH] = PATTERN_SQUID;
    char log_regexp_combined_enhanced[MAX_RE_LENGTH] = PATTERN_COMBINED_ENHANCED;
    char log_regexp_iptables[MAX_RE_LENGTH] = PATTERN_IPTABLES;
    char log_regexp_syslog[MAX_RE_LENGTH] = PATTERN_SYSLOG;

    const char *error;                          /* RE error pointer, offset */
    int erroffset;                              /* RE error value */

    /* CLF */
    cmp_log_regexp_clf = pcre_compile(log_regexp_clf, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile CLF%s", "\n")
        if (cmp_log_regexp_clf == NULL) {
        re_compile_failed(erroffset, error, log_regexp_clf);
    }

    /* Enhanced Combined */
    cmp_log_regexp_combined_enhanced = pcre_compile(log_regexp_combined_enhanced, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile COMBINED_ENHANCED%s", "\n")
        if (cmp_log_regexp_combined_enhanced == NULL) {
        re_compile_failed(erroffset, error, log_regexp_combined_enhanced);
    }

    /* FTP XFERLOG */
    cmp_log_regexp_xferlog = pcre_compile(log_regexp_xferlog, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile PATTERN_XFERLOG%s", "\n")
        if (cmp_log_regexp_xferlog == NULL) {
        re_compile_failed(erroffset, error, log_regexp_xferlog);
    }

    /* SQUID LOG */
    cmp_log_regexp_squid = pcre_compile(log_regexp_squid, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile PATTERN_SQUID%s", "\n")
        if (cmp_log_regexp_squid == NULL) {
        re_compile_failed(erroffset, error, log_regexp_squid);
    }

    /* SYSLOG/IPTABLES LOG */
    cmp_log_regexp_iptables = pcre_compile(log_regexp_iptables, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile PATTERN_IPTABLES%s", "\n")
        if (cmp_log_regexp_iptables == NULL) {
        re_compile_failed(erroffset, error, log_regexp_iptables);
    }

    /* SYSLOG */
    cmp_log_regexp_syslog = pcre_compile(log_regexp_syslog, 0, &error, &erroffset, NULL);
    VPRINT(VERBOSE2, "PCRE: Compile PATTERN_SYSLOG%s", "\n")
        if (cmp_log_regexp_syslog == NULL) {
        re_compile_failed(erroffset, error, log_regexp_syslog);
    }
}


/************************************************************************
 ************************************************************************
 *                      END OF FILE                                     *
 ************************************************************************/