ingest_stats.c

Go to the documentation of this file.

/*******************************************************************************
*
*  COPYRIGHT (C) 2006 Battelle Memorial Institute.  All Rights Reserved.
*
********************************************************************************
*
*  Author:
*     name:  Brian Ermold
*     phone: (509) 375-2277
*     email: brian.ermold@pnl.gov
*
********************************************************************************
*
*  RCS INFORMATION:
*    $RCSfile: ingest_stats.c,v $
*    $Revision: 1.5 $
*    $Locker:  $
*    $Date: 2006/08/04 02:23:46 $
*    $State: Exp $
*
********************************************************************************
*
* DESCRIPTION:
*
*   The <b>set_data_stats</b> function should be called when a Zebra Datachunk
*   is stored. Information about the start time, end time, and number of samples
*   in the datachunk is then recorded in a static structure. When the program is
*   exiting it should then use <b>write_data_stats</b> to write all the gather
*   information to the log file using the <b>append_instrlog</b> function. The
*   following lines will be added to the log file for each platform found:
*     <blockquote><p><b>
*       ${Platform Name} Samples: ${Number Of Samples}<br>
*       ${Platform Name} Begin Time: ${Begin Time}<br>
*       ${Platform Name} End Time: ${End_time}</b></p>
*     </blockquote><p>
*   The <b>set_file_stats</b> function should be called after a file has been
*   processed and moved to the ingested directory. When called it will print the
*   following line to the log file using the <b>append_instrlog</b> function:
*     </p><blockquote><p><b>
*       Processed ${filename}</b></p>
*     </blockquote><p>
*   Information is then gathered about this file and stored in a static
*   structure containing the size of the file and wether it was good or bad
*   (based on wether ".bad." was found in the file name or not). When the
*   program is exiting it should then call <b>write_file_stats</b> wich will
*   write all the gathered information to the log file using the
*   <b>append_instrlog</b> function. The following lines will be added to the
*   log file:</p><blockquote><p><b>
*       Total Bytes: ${gFStats.tot_bytes}<br>
*       Good Bytes: ${gFStats.good_bytes}<br>
*       Bad Bytes: ${gFStats.bad_bytes}<br>
*       Total Files: ${gFStats.tot_files}<br>
*       Good Files: ${gFStats.good_files}<br>
*       Bad Files: ${gFStats.bad_files} </b> </p>
*   </blockquote>
*
* END DESCRIPTION
*******************************************************************************/

static char *rcsid = "$Id: ingest_stats.c,v 1.5 2006/08/04 02:23:46 ermold Exp $";

#include <ctype.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>

#include "ingest_lib/ingest_stats.h"
#include "ingest_lib/sds_util.h"
#include "ingest_lib/sds_log.h"
#include "ingest_lib/sds_tdb.h"

/*******************************************************************************
*  Private Variables:
*/

static FileStats gFStats;  /* The structure to store the file stats      */
static DcStats   gDcStats; /* The structure to store the datachunk stats */

/* End Private Variables */

/*******************************************************************************
*
* Description:
*   The reset_file_stats function will reset the values to zero of the file
*   stats structure.
*
* Inputs:
*   None.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void reset_file_stats(void)
{
    gFStats.tot_bytes = gFStats.good_bytes = gFStats.bad_bytes = 0;
    gFStats.tot_files = gFStats.good_files = gFStats.bad_files = 0;
}

/*******************************************************************************
*
* Description:
*   The set_file_stats function will increment the current file stats with
*   information obtained from the specified file.
*
* Inputs:
*   filename:  The filename for which stats should be added.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void set_file_stats(char *filename)
{
    struct stat stat_buf;
    off_t       bytes;

    if ((filename == NULL) || (filename[0] == '\0')) {
        print_debug(__FILE__, __LINE__,
            "No filename was specified in set_file_stats\n");
    }
    else {
        print_debug(__FILE__, __LINE__,
            "Getting file stats for: %s\n", filename);

        /*  Appened this filename to the instr_log
         */

        append_instrlog("Processed %s\n", filename);

        if (stat(filename, &stat_buf) == 0) {
            bytes = stat_buf.st_size;
        }
        else {
            bytes = 0;
            append_log_msg(__FILE__, __LINE__,
                "Error #%i getting stats for file: %s\n -> %s\n",
                errno, filename, strerror(errno));
        }

        gFStats.tot_files++;
        gFStats.tot_bytes += bytes;

        if (strstr(filename, ".bad.")) {
            gFStats.bad_files++;
            gFStats.bad_bytes += bytes;
        }
        else {
            gFStats.good_files++;
            gFStats.good_bytes += bytes;
        }
    }
}

/*******************************************************************************
*
* Description:
*   The write_file_stats function will write the current file stats to the
*   log file.
*
* Inputs:
*   None.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void write_file_stats(void)
{
    append_instrlog("\nTotal Bytes: %d\n", gFStats.tot_bytes);
    append_instrlog("Good Bytes:  %d\n", gFStats.good_bytes);
    append_instrlog("Bad Bytes:   %d\n", gFStats.bad_bytes);
    append_instrlog("Total Files: %d\n", gFStats.tot_files);
    append_instrlog("Good Files:  %d\n", gFStats.good_files);
    append_instrlog("Bad Files:   %d\n", gFStats.bad_files);
}

/*******************************************************************************
*
* Description:
*   The reset_dc_stats function will reset to zero the value of the
*   data stats struct.
*
* Inputs:
*   None.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void reset_dc_stats(void)
{
    int i;

    gDcStats.tot_samples = 0;

    for(i = 0; i < MAX_DCS; i++) {
        gDcStats.plat_name[i][0]           = '\0';
        gDcStats.plat_samples[i]           = 0;
        gDcStats.begin_time[i].zt_Sec      = 0;
        gDcStats.begin_time[i].zt_MicroSec = 0;
        gDcStats.end_time[i].zt_Sec        = 0;
        gDcStats.end_time[i].zt_MicroSec   = 0;
    }
}

/*******************************************************************************
*
* Description:
*   The set_dc_stats function will increment the current data stats with
*   information obtained from the specified DataChunk.
*
* Inputs:
*   dc:   The DataChunk for which stats should be added.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void set_dc_stats(DataChunk *dc)
{
    int     nsamples;
    int     pindex;
    ZebTime when;

    if (dc == NULL) {
        print_debug(__FILE__, __LINE__,
            "No datachunk was specified in set_dc_stats\n");
    }
    else {

        /*  Get the platform index
         */

        pindex = set_pindex(dc);

        /*  Get the number of samples and add this up
         */

        nsamples = dc_GetNSample(dc);
        gDcStats.tot_samples          += nsamples;
        gDcStats.plat_samples[pindex] += nsamples;

        /*  Store the name of the platform
         */

        strcpy(gDcStats.plat_name[pindex], ds_PlatformName(dc->dc_Platform));

        /*  Set the first sample time
         */

        dc_GetTime(dc, 0, &when);
        if ((when.zt_Sec < gDcStats.begin_time[pindex].zt_Sec)
            || (gDcStats.begin_time[pindex].zt_Sec == 0)) {
            gDcStats.begin_time[pindex] = when;
        }

        /*  Set the last sample time
         */

        dc_GetTime(dc, nsamples - 1, &when);
        if ((when.zt_Sec > gDcStats.end_time[pindex].zt_Sec)
            || (gDcStats.end_time[pindex].zt_Sec == 0)) {
            gDcStats.end_time[pindex] = when;
        }
    }
}

static void parse_datastream_name(char *ds_name,
    char *ds_site, char *ds_base, char *ds_fac, char *ds_level)
{
    int i, j;
    int site_length;

    /* Example datastream name: sgp5ebbrE12.b1 */

    /* Site */

    site_length = (ds_name[0] == 'D') ? 4 : 3;

    for (i = 0; i < site_length; i++) ds_site[i] = ds_name[i];
    ds_site[i] = '\0';

    /* Datastream Base Name */

    for (j = 0; !isupper(ds_name[i]); i++, j++) ds_base[j] = ds_name[i];
    ds_base[j] = '\0';

    /* Facility */

    for (j = 0; ds_name[i] != '.'; i++, j++) ds_fac[j] = ds_name[i];
    ds_fac[j] = '\0';

    /* Datastream Level */

    i++; /* skip the "dot" */

    for (j = 0; ds_name[i] != '\0'; i++, j++) ds_level[j] = ds_name[i];
    ds_level[j] = '\0';
}


/*******************************************************************************
*
* Description:
*   The write_dc_stats function will write the current data stats to the log file.
*
* Inputs:
*   None.
*
* Outputs:
*   None.
*
* Returns:
*   None.
*
*******************************************************************************/
void write_dc_stats()
{
    time_t zt_Sec;
    char   ds_site[8];
    char   ds_base[32];
    char   ds_fac[8];
    char   ds_level[8];
    int    dcs;

    append_instrlog("\nTotal Samples: %ld\n", gDcStats.tot_samples);

    for(dcs = 0; dcs < MAX_DCS; dcs++) {

        if (gDcStats.plat_samples[dcs]) {
            append_instrlog("%s Samples: %d\n", gDcStats.plat_name[dcs],
                gDcStats.plat_samples[dcs]);

            zt_Sec = gDcStats.begin_time[dcs].zt_Sec;

            append_instrlog("%s Begin Time: %s", gDcStats.plat_name[dcs],
                asctime(gmtime(&zt_Sec)));

            zt_Sec = gDcStats.end_time[dcs].zt_Sec;

            append_instrlog("%s End Time:   %s", gDcStats.plat_name[dcs],
                asctime(gmtime(&zt_Sec)));

            parse_datastream_name(gDcStats.plat_name[dcs],
                ds_site, ds_base, ds_fac, ds_level);

            update_datastream_times(
                ds_base,
                ds_level,
                gDcStats.begin_time[dcs].zt_Sec,
                gDcStats.end_time[dcs].zt_Sec);
        }
    }
}

---