#!/bin/sh
#set -x   ## uncomment for a trace
# (tabs are 3-spaces in width)

# 2023-Oct-26
# This material is free and without warranty.  Use it in any way you think fits.

#+++++ START: USER CONFIGURATION AREA ++++++++++++++++++++++++++++++++++++++++++
#-------------------------------------------------------------------------------
#
# define the location of the text editor you want to use to edit the 
#TEXT_EDITOR=''  # use the default /Applications/TextEditor (must be plain text)
TEXT_EDITOR='/Apps/Text Tools/Editors/BBEdit/BBEdit.app'
#
#-------------------------------------------------------------------------------
#
# NOTEs:
# - to determine the CSV file's list of fields, run the script/command:
#   extractCSVdata.sh -iq <CSV pathname>
# - to determine the list of formulas that can be used to process CSV fields,
#   run the script/command:
#   extractCSVdata.sh -fq <CSV pathname>
#
#-------------------------------------------------------------------------------
#
# Terminology:
# - the lines in a CSV file are called records
# - the comma-separated data-elements that make up a record are called fields
# - the record that provides the names of the fields is the header record
#
# CSV_FIELDS_<fields ID> specifications:
# - multiple "CSV_FIELDS" specifications are allowed as
#   "CSV_FIELDS_<fields ID>" where:
#   * different <fields ID> values can be appended to "CSV_FIELDS_" to create
#     various CSV_FIELDS specifications - e.g., for different types of CSV files
#     and/or different variants of the same CSV file types
#   * the <fields ID> must only have alphabetic, alphanumeric and/or
#     underscore (_) characters (others will cause script errors)
#
# CSV_FIELDS_<fields ID> specifications can be used to specify the CSV fields to
# be extracted -- the format is:
#
# <field name part>[,[<new field name>][,[<format spec>][,[<required value flag>][,[<formula number>][#<comment>]]]]]
# <ignored entries>
#
# Where:
# a comma (,) character separates the different fields within an entry
#
# <field name part> is the name of a field in the header record in the CSV file:
#                   - a "matching" name-segment is sufficient -- the last
#                     CSV field that has the "best" left-most match to that
#                     name-segment, when searching left-to-right through each of
#                     the source CSV field names, is selected
#                   - the "matching" is a "best match" that is:
#                     * an exact match OR
#                     * the left-most match OR
#                     * the left-most match AND where the <field name part> is
#                       the greatest portion of the source CSV field name
#                   - e.g., for the source CSV file's header record segment:
#                           ...,WheelSpeed,Speed,...
#                     * the <field name part> 'Speed' would match the "Speed"
#                       source CSV field because 'Speed' is an exact match
#                   - e.g., for the source CSV file's header record segment:
#                           ...,WheelSpeed,gpsSpeed,...
#                     * the <field name part> 'Speed' would match the "gpsSpeed"
#                       source CSV field because 'Speed' matches "gpsSpeed" at
#                       the 4th character from the left but not until the 5th
#                       character from the left for "WheelSpeed"
#                   - e.g., for the source CSV file's header record segment:
#                           ...,theGsum,theGsumMax,theGsumPercent,...
#                     * the <field name part> 'Gsum' would match the "theGsum"
#                       source CSV field because 'Gsum" matches at same/4th from
#                       the left character in "theGsum", "theGsumMax" and
#                       "theGsumPercent" but 'Gsum' is a greater portion of
#                       "theGsum" than it is of either "theGsumMax" or
#                       "theGsumPercent"
#                   - the names/name-segments are case-sensitive
#                   - entries missing this field are ignored
#                   - single quote ('), tilde (~) and question mark (?)
#                     characters are not allowed:
#                     * single quotes are ignored/removed for processing
#                     * tildes and question marks are changed to dashes (-)
#                       characters for processing
#                   - the -i and -iq options will show the resulting field
#                     selections (and more)
#
# <new field name> is the (optional) new/different name for this field that is
#                  to be used in the header record for the result file:
#                  - an empty <new field name> entry means that the entry in the
#                    <field name part> will be used
#                  - a single bang/exclamation (!) entry means that the field
#                    name from the source CSV file will be used
#
# <format spec> is a printf-style format specification that will be used to
#               format that field's data when extracted into the results CSV
#               file -- an empty <format spec> defaults to the %s format
#
# <required-value flag> is 1 if that CSV field is required to have a value --
#                       i.e., if that field does not have a value, then that
#                             entire source CSV record is omitted, but is still
#                             processed to collect previous-field values when
#                             copy-forward field-filling is invoked and/or
#                             forumlas are being applied
#
# <sort-order number> is 0 if that result field is not to be used to sort the
#                     non-header records or a positive integer that specifies
#                     the priority of that field when sorting the records:
#                     - sorting occurs using the field values from the source
#                       CSV records immediately after the result fields are
#                       extracted and before any other processing begins: i.e.,
#                       if there are 3 fields that are to be used for sorting,
#                       then specifying values of 1 for field 10, 3 for field
#                       2 and 5 for field 4 would cause the sorting to occur
#                       by fields 10 then 2 then 4
#                     (the entry value specifies the order of the sort fields)
#
# <formula number> is the number of the formula that specifies the processing to
#                  be performed to compute that field's value before it is
#                  written to the results file -- an empty or zero (0) entry
#                  means that no formula is applied and no processing takes
#                  place
#
# #<comment> is a comment at the end of a fields-spec entry -- whitespace that
#            precedes the hash (#) character, the hash character and all content
#            that follows the hash character is ignored during processing
#
# <ignored entries>
# - comment entries, where the first non-whitespace character is a hash (#), are
#   ignored/removed during processing
# - empty/whitespace-only entries are ignored/removed during processing
# - entries with an empty/whitespace-only <field name part> are ignored/removed
#   during processing
#
# NOTEs:
# - tab characters are changed to a single space character
# - whitespace at the start/end of lines & adjacent to commas is ignored/removed
# - single quotes are ignored/removed
# - don't forget adjacent commas for empty fields that have a subsequent field
#
# ==============================================================================
# Extract all the fields in my RaceCapture CSV/log file and add a new field that
# contains human-readable local times converted from the Utc epoch number field
CSV_FIELDS_RCaddLocalTime='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
    Interval , Interval     ,     %10d ,  1  ,     ,  
         Utc , UTC          ,     %14d ,  1  ,  1  ,  
         Utc , LocalTime    ,     %24s ,  0  ,     ,  73
#
      AccelX , AccelX       ,   %+6.2f ,  1  ,     ,  # my mounting makes this Y
      AccelY , AccelY       ,   %+6.2f ,  1  ,     ,  # my mounting makes this X
      AccelZ , AccelZ       ,   %+6.2f ,  1  ,     ,  
         Yaw , Yaw          ,   %+7.2f ,  0  ,     ,  
       Pitch , Pitch        ,   %+7.2f ,  0  ,     ,  
        Roll , Roll         ,   %+7.2f ,  0  ,     ,  
       Gsum , Gsum         ,    %6.2f ,  0  ,     ,  
     GsumMax , GsumMax      ,    %6.2f ,  0  ,     ,  
     GsumPct , GsumPct      ,      %4d ,  0  ,     ,  
#
    Latitude , Latitude     ,  %+12.7f ,  0  ,     ,  
   Longitude , Longitude    ,  %+13.7f ,  0  ,     ,  
       Speed , Speed        ,    %7.2f ,  0  ,     ,  # speed computed via GPS
    Altitude , Altitude     ,   %+9.2f ,  0  ,     ,  
    Distance , Distance     ,    %9.4f ,  0  ,     ,  
     GPSSats , GPSSats      ,      %2d ,  0  ,     ,  
     GPSQual , GPSQual      ,      %2d ,  0  ,     ,  
      GPSDOP , GPSDOP       ,    %4.1f ,  0  ,     ,  
#
  WheelSpeed , WheelSpeed   ,    %7.2f ,  0  ,     ,  
         RPM , RPM          ,      %5d ,  0  ,     ,  
         TPS , TPS          ,      %3d ,  0  ,     ,  
  EngineLoad , EngineLoad   ,    %5.1f ,  0  ,     ,  
  EngineTemp , EngineTemp   ,    %5.1f ,  0  ,     ,  
   FuelLevel , FuelLevel    ,      %3d ,  0  ,     ,  
      Beeper , Beeper       ,      %1d ,  0  ,     ,  
#
 SessionTime , SessionTime  ,    %9.4f ,  0  ,     ,  
    LapCount , LapCount     ,      %4d ,  0  ,     ,  
     LapTime , LapTime      ,    %8.4f ,  0  ,     ,  
  CurrentLap , CurrentLap   ,      %4d ,  0  ,     ,  
 ElapsedTime , ElapsedTime  ,   %11.4f ,  1  ,     ,  # current lap elapsed time
    PredTime , PredTime     ,    %8.4f ,  0  ,     ,  
      Sector , Sector       ,      %2d ,  0  ,     ,  
  SectorTime , SectorTime   ,    %8.4f ,  0  ,     ,  
'
#
# ==============================================================================
# THESE "..._Car_..." & "..._Timing_..." FIELDS SPECS ARE USED BY splitRClog.sh
#
# Extract car-related fields in various versions of RaceCapture CSV/log files
# for use in RaceRender (splitting the car/timing data improves editing
# performance and allows the car/OBD-queried data to be synchronized separately)
#
# For unprocessed current RaceCapture CSV files from GTR
CSV_FIELDS_Car_GTR='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         Utc , UTC          ,     %14d ,  1  ,  1  ,  
         Utc , LocalTime    ,     %24s ,  0  ,     ,  73
#
      AccelX , AccelX       ,   %+6.2f ,  1  ,     ,  # my mounting makes this Y
      AccelY , AccelY       ,   %+6.2f ,  1  ,     ,  # my mounting makes this X
      AccelZ , AccelZ       ,   %+6.2f ,  1  ,     ,  
        Gsum , Gsum         ,    %6.2f ,  0  ,     ,  
#
  WheelSpeed , WheelSpeed   ,    %7.2f ,  0  ,     ,  
         RPM , RPM          ,      %5d ,  0  ,     ,  
         TPS , TPS          ,      %3d ,  0  ,     ,  
  EngineLoad , EngineLoad   ,    %5.1f ,  0  ,     ,  
  EngineTemp , EngineTemp   ,    %5.1f ,  0  ,     ,  
   FuelLevel , FuelLevel    ,      %3d ,  0  ,     ,  
'
#
# For current RaceCapture CSV files from GTR after being processed using the
# CSV_FIELDS_RCaddLocalTime fields specification
CSV_FIELDS_Car_GTRlts='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         UTC , UTC          ,     %14d ,  1  ,  1  ,  
   LocalTime , LocalTime    ,     %24s ,  0  ,     ,  
#
      AccelX , AccelX       ,   %+6.2f ,  1  ,     ,  # my mounting makes this Y
      AccelY , AccelY       ,   %+6.2f ,  1  ,     ,  # my mounting makes this X
        Gsum , Gsum         ,    %6.2f ,  0  ,     ,  
#
  WheelSpeed , WheelSpeed   ,    %7.2f ,  0  ,     ,  
         RPM , RPM          ,      %5d ,  0  ,     ,  
         TPS , TPS          ,      %3d ,  0  ,     ,  
  EngineLoad , EngineLoad   ,    %5.1f ,  0  ,     ,  
  EngineTemp , EngineTemp   ,    %5.1f ,  0  ,     ,  
   FuelLevel , FuelLevel    ,      %3d ,  0  ,     ,  
'
#
# ------------------------------------------------------------------------------
# Extract timing-related fields in various versions of RaceCapture CSV/log files
# for use in RaceRender (splitting the car/timing data improves editing
# performance and allows the car/OBD-queried data to be synchronized separately)
#
# For unprocessed current RaceCapture CSV file from GTR
CSV_FIELDS_Timing_GTR='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         Utc , UTC          ,     %14d ,  1  ,  1  ,  
         Utc , LocalTime    ,     %24s ,  0  ,     ,  73
#
    Latitude , Latitude     ,  %+12.7f ,  0  ,     ,  
   Longitude , Longitude    ,  %+13.7f ,  0  ,     ,  
       Speed , Speed        ,    %7.2f ,  0  ,     ,  # speed computed via GPS
    Altitude , Altitude     ,   %+9.2f ,  0  ,     ,  
     GPSSats , GPSSats      ,      %2d ,  0  ,     ,  
     GPSQual , GPSQual      ,      %2d ,  0  ,     ,  
      GPSDOP , GPSDOP       ,    %4.1f ,  0  ,     ,  
#
    LapCount , LapCount     ,      %4d ,  0  ,     ,  
     LapTime , LapTime      ,    %8.4f ,  0  ,     ,  
  CurrentLap , CurrentLap   ,      %4d ,  0  ,     ,  
 ElapsedTime , ElapsedTime  ,   %11.4f ,  1  ,     ,  # current lap elapsed time
    PredTime , PredTime     ,    %8.4f ,  0  ,     ,  
'
#
# For current RaceCapture CSV file from GTR after being processed using the
# CSV_FIELDS_RCaddLocalTime fields specification
CSV_FIELDS_Timing_GTRlts='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         UTC , UTC          ,     %14d ,  1  ,  1  ,  
   LocalTime , LocalTime    ,     %24s ,  0  ,     ,  
#
    Latitude , Latitude     ,  %+12.7f ,  0  ,     ,  
   Longitude , Longitude    ,  %+13.7f ,  0  ,     ,  
       Speed , Speed        ,    %7.2f ,  0  ,     ,  # speed computed via GPS
    Altitude , Altitude     ,   %+9.2f ,  0  ,     ,  
     GPSSats , GPSSats      ,      %2d ,  0  ,     ,  
     GPSQual , GPSQual      ,      %2d ,  0  ,     ,  
      GPSDOP , GPSDOP       ,    %4.1f ,  0  ,     ,  
#
    LapCount , LapCount     ,      %4d ,  0  ,     ,  
     LapTime , LapTime      ,    %8.4f ,  0  ,     ,  
  CurrentLap , CurrentLap   ,      %4d ,  0  ,     ,  
 ElapsedTime , ElapsedTime  ,   %11.4f ,  1  ,     ,  # current lap elapsed time
    PredTime , PredTime     ,    %8.4f ,  0  ,     ,  
'
#
#-------------------------------------------------------------------------------
# extract RaceCapture fields for use as car/OBD-related data in RaceRender
CSV_FIELDS_GTRodb='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         Utc , UTC       ,     %14d ,  1  ,  1  ,  
         Utc , LocalTime ,     %24s ,  0  ,     ,  73
#
  WheelSpeed ,           ,    %7.2f ,  0  ,     ,  
         RPM ,           ,      %5d ,  0  ,     ,  
         TPS ,           ,      %3d ,  0  ,     ,  
  EngineLoad ,           ,    %5.1f ,  0  ,     ,  
  EngineTemp ,           ,    %5.1f ,  0  ,     ,  
   FuelLevel ,           ,      %3d ,  0  ,     ,  
'
#
#-------------------------------------------------------------------------------
# extract fields for use as accelerometer-related data in RaceRender
CSV_FIELDS_GTRaccel='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         Utc ,    UTC       ,     %14d ,  1  ,  1  ,  
         Utc ,    LocalTime ,     %24s ,  0  ,     ,  73
#   
      AccelX ,              ,   %+6.2f ,  0  ,     ,  
      AccelY ,              ,   %+6.2f ,  0  ,     ,  
      AccelZ ,              ,   %+6.2f ,  0  ,     ,  
        Gsum ,              ,    %6.2f ,  0  ,     ,  
     GsumMax ,              ,    %6.2f ,  0  ,     ,  
     GsumPct ,              ,      %4d ,  0  ,     ,  
         Yaw ,              ,   %+7.2f ,  0  ,     ,  
       Pitch ,              ,   %+7.2f ,  0  ,     ,  
        Roll ,              ,   %+7.2f ,  0  ,     ,  
'
#
#-------------------------------------------------------------------------------
# extract fields for use as timing-related data in RaceRender
CSV_FIELDS_GTRtiming='
# CSV field-     results        format   is   sort   formula
# name part     field name       spec   req"d order  number
#------------  -------------  --------- ----- ----- --------
         Utc ,    UTC       ,     %14d ,  1  ,  1  ,  
         Utc ,    LocalTime ,     %24s ,  0  ,     ,  73
#   
    Latitude ,              ,  %+12.7f ,  1  ,     ,  
   Longitude ,              ,  %+13.7f ,  1  ,     ,  
       Speed ,              ,    %7.2f ,  0  ,     ,  
    Altitude ,              ,   %+9.2f ,  0  ,     ,  
    Distance ,              ,    %9.4f ,  0  ,     ,  
     GPSSats ,              ,      %2d ,  0  ,     ,  
     GPSQual ,              ,      %2d ,  0  ,     ,  
      GPSDOP ,              ,    %4.1f ,  0  ,     ,  
 #   
 SessionTime ,              ,    %9.4f ,  0  ,     ,  
    LapCount ,              ,      %4d ,  0  ,     ,  
     LapTime ,              ,    %8.4f ,  0  ,     ,  
  CurrentLap ,              ,      %4d ,  0  ,     ,  
 ElapsedTime ,              ,   %11.4f ,  0  ,     ,  
    PredTime ,              ,    %8.4f ,  0  ,     ,  
      Sector ,              ,      %2d ,  0  ,     ,  
  SectorTime ,              ,    %8.4f ,  0  ,     ,  
'
#
#-------------------------------------------------------------------------------
# examples of some time conversions and how to capture 2 field values and create
# a third field with the value that is the average of the 2 other field values
CSV_FIELDS_example='
# CSV field-    results      format   is   sort   formula
# name part    field name     spec   req"d order  number
#------------  ----------  --------- ----- ----- --------
# the supplied Interval is the number of msec the data-logger has been running
    Interval , UpTime      ,     %13s ,  0  ,  1  ,  66   # convert msec to H:MM:SS.sss
#
# the ElapsedTime is the number of minutes that have elapsed, with 4 decimals
 ElapsedTime , ElapsedTime ,   %11.4f ,  0  ,     ,  
 ElapsedTime , ET/H:M:S.sss,     %13s ,  0  ,     ,  67   # convert M.n to H:MM:SS.sss 
#
       Speed , GPSspeed    ,    %7.2f ,  0  ,     ,  300  # capture field value as sf[1]
  WheelSpeed , WheelSpeed  ,    %7.2f ,  0  ,     ,  301  # capture field value as sf[1]
       Speed , AvgSpeed    ,    %7.2f ,  0  ,     ,  310  # replace placeholder field w/average
#
    Latitude , Latitude    ,  %+12.7f ,  1
   Longitude , Longitude   ,  %+13.7f ,  1
#
    Interval , !         ,     %12d   ,  1  ,     ,       # use the source CSV field name
'
#
#-------------------------------------------------------------------------------
#
# optionally, define the location of a file containing the collection of
# fields-spec entries ... if non-empty, it overrides CSV_FIELDS_<fields ID>
# entries, above, but is itself overridden by a "-fsp <fields spec pathname>"
# script argument, so the hierarchy, with highest priority first, is:
# - filename via -fsp argument
# - filename set in THE_CSV_FIELDS_SPEC_FILE variable
# - field specs that are defined as CSV_FIELDS_<fields ID> variables, above
#
#THE_CSV_FIELDS_SPEC_FILE='' # NOTE: variable name can't be CSV_FIELDS_SPEC_FILE
THE_CSV_FIELDS_SPEC_FILE=`/usr/bin/dirname "$0"`'/CSVfieldSpecs.txt'
#
#-------------------------------------------------------------------------------
#
# Formulas:
# These formulas/computations can be applied to a field as it is extracted and
# included in a results file via extractCSVdata.sh
#
# The formula-entry format is:
#   <ID number>~<formula name>~<formula>
#   <ignored entries>
#
#  Where:
# a tilde (~) character separates the different entry fields
#
# <ID number> is a positive integer with maximum length of 4 where the order
#             of the ID numbers has no computational significance
#             (leading/trailing whitespace is ignored/removed)
#
# <formula name> is a human-readable (hopefully meaningful) name that describes
#                the formula (leading/trailing whitespace is ignored/removed)
#
# <formula> is the computational/math formula used to compute/capture a value
#           (leading/trailing whitespace is ignored/removed)
#
# <ignored entries>
# - comment entries, where the first non-whitespace character is a hash (#), are
#   ignored/removed during processing
# - empty/whitespace-only entries are ignored/removed during processing
# - entries with an empty/whitespace-only <ID number>, <formula name> and/or
#   <formula> are ignored/removed during processing
#
# - formulas can use the following pre-defined variables:
#   (a processed value is the result after a formula has been applied)
#   * curr is automatically assigned the current value for the field being
#          processed (this is the current field value from the source CSV file)
#   * prev is automatically assigned the previous value from the source CSV file
#          for the field being processed -- NOTE that this is the unprocessed
#          previous value for the field
#   * prior is a value that can automatically be assigned the previously
#           computed value for the field being processed -- it can be used only
#           in conjunction with the b4() "pseudo-function" (see more below)
#   * tzo is the user-supplied or system-supplied timezone offset from UTC, in
#         seconds
#
# - formulas can use the provided rounding function "rd()" to control the number
#   of decimal places in the output (also see notes on scale, below):
#      rd(value, decimals)
#   e.g., rd(123.456789, 3) yields 123.457
#
# - formulas can also be one of multiple built-in functions:
#   * the built-in functions will cause the field to be converted and output
#     without being additionally processed
#   * only 1 built-in function can appear in any given formula entry
#   * additional formula statements in a formula entry that includes a built-in
#     formula will be ignored
#
# - built-in time-conversion functions:
#   * eNumToUTCdateTime(); convert the field value from a UNIX-style epoch
#                          number to a UTC date/time as YYYY-MM-DD HH:MM:SS
#   * eNumToLocalDateTime(); convert the field value from a UNIX-style epoch
#                            number to a date/time in the timezone specified
#                            by the timezone offset as YYYY-MM-DD HH:MM:SS
#   * eNumToUTCdateTimeMsec(); convert the field value from a UNIX-style epoch
#                              number that includes milliseconds to a UTC
#                              date/time as YYYY-MM-DD HH:MM:SS.sss
#   * eNumToLocalDateTimeMsec(); convert the field value from a UNIX-style
#                                epoch number that includes milliseconds to a
#                                date/time in the timezone specified by the
#                                time-zone offset as YYYY-MM-DD HH:MM:SS.sss
#   * UTCdateTimeTOeNum(); convert the field value from a UTC date/time in
#                          YYYY-MM-DD HH:MM:SS or YYYY-MM-DD HH:MM:SS.sss
#                          format to a UNIX-style epoch number
#   * localDateTimeTOeNum(); convert the field value from a date/time in
#                            YYYY-MM-DD HH:MM:SS or YYYY-MM-DD HH:MM:SS.sss
#                            format in the timezone specified by the time-zone
#                            offset to a UNIX-style epoch number
#   * tDT(); transpose a date/time "timestamp" from one format to another -- see
#            the detailed documentation along with some examples in the supplied
#            formulas below (or in the supplied formulas file)
#
# - formulas can also use a special b4() "pseudo-function" to indicate the
#   portion of the formula that computes the final value that is the result
#   field and is also to be saved as that field's prior (previously computed) so
#   the value can be used when processing that field in the next record
#   value:
#   e.g., to perform a simple exponential data smoothing, the formula is:
#         a=0.75; b4((a * curr) + ((1 - a) * prior))
#   where:
#   * a : is the "alpha" smoothing-level factor (0 to 1, 1 yields no smoothing)
#   * (a * curr) + ((1 - a) * prior) : is the part of the formula that generates
#                                      the smoothed value (the "result")
#   * b4 : is the "pseudo-function" that wraps the smoothing formula and causes
#          the result to also be made available as the next record's prior value
#   * prior : is the field's value that was computed for the previous record --
#             it's initial value is set to the first non-empty value for that
#             field
#
# - formulas can have user-defined variables, other than curr/prev/prior/tzo:
#   * when using formulas, the entire output is streamed through the bc utility
#     "as one" ... this means that, once assigned a value, a variable holds that
#     value until a different value is assigned (-sf option shows input to bc)
#   * it is possible to capture the value of fields in order to do calculations
#     that use other-field values (see the fields spec with suffix "example")
#
# - composing formulas:
#   * formulas consist of a series of statements where each statement is
#     terminated by a semi-colon (;) character -- statements are things like
#     variable/value assignments, computational formulas and quoted/text output
#   * formulas can also have conditional and loop statements
#   * the b4() "pseudo-function" can only contain assignable expressions -- so,
#     for example, no conditional, loop or print statements inside a b4()
#   * if using division, division by zero must not occur (or things will fail!)
#   * a tab character is changed to a single space character during processing
#   * extra spaces around a formula are allowed but ignored during processing
#   * single quotes (') and question marks (?) are NOT allowed and are removed
#     during processing
#   * formula comments (valid in the bc utility) are NOT allowed inside formulas
#   * the bc utility has some unusual operator precedence so being explicit with
#     parenthesis is safest (and is always the most clear, at any time!)
#   * bc has a reasonable set of operators and trigonometry functions
#   * ultimately, the formulas are computed via the "bc -l" utility and must
#     conform to bc's syntax and semantics (see bc docs via "man bc" command)
#
# - all calculations are done using arbitrary precision decimal where the scale
#   is the total number of decimal digits after the decimal point and the length
#   is the total number of significant decimal digits:
#   * see above notes on rd() function for controlling number of decimals in the
#     output, but note that scale may be required in some cases -- e.g., when
#     using the modulo (%) operator with integers
#   * the scale is set to twenty (20) for the start of each applied formula
#
# - "scale=#;" can be added to a formula, where # is a positive integer:
#   * the scale value sets the number of digits after the decimal point to which
#     calculations -- including intermediate calculations -- and results will be
#     performed and presented ... be careful; e.g., if scale=0 and _any_ of the
#     intermediate calculations are <0, the result will always be 0
#   * each "scale=#" entry applies to the end of the formula or the next scale
#   * since the order of formulas during the calculations is undefined, the use
#     of scale must be explicit for each formula, if you want to vary the scale
#   * a constant, in a formula, will override/change the scale to the number of
#     digits that are supplied after the decimal point in the constant
#   * a "no-op" divide by 1 can used to apply the scale value, but note that
#     when scale limits the length of a value, it truncates and does NOT round
#   * scale can be "tricky" ... if in doubt, manually test via command line --
#     e.g.:
#     echo "a=10; b=5; scale=1; b/a; b/10.00; (5.00/a)*1.00; scale=3; b/a" | bc
#     yields: .5, .5, .50 and .500
#   * for details, see the documentation for the bc utility; i.e., use "man bc"
#
# - any/all newlines output by calculations are removed
#
# - non-numeric output can be accomplished in 2 ways:
#   * a string is text enclosed in double-quotes -- e.g.: "this is a string";
#   * strings are simply passed through to the output -- a string must be
#     terminated as a statement (i.e., with a semi-colon)
#   * text and variable outputs can be output via the "print list" statement
#     where the list is a comma-separated collection of strings and variables
#   e.g.,     this:  echo 'h=10; m=2;  h;":"; if(m<10)"0"; m;' | bc
#         and this:  echo 'h=10; m=2;  print h,":"; if(m<10) print "0"; m;' | bc
#         with both yield "10:02" (after formula-generated newlines are removed)
#
# - arrays can be used to collect multiple results to output -- e.g.:
#   r[1]=curr; r[2]=prev; print "curr = ",r[1],", prev = ",r[2]
#
# - non-conforming formula entries are likely to generate errors like this:
#   (standard_in) 54: parse error
#   (standard_in) 108: parse error
#   (standard_in) 177: parse error
#   ...
#
# IMPORTANT NOTEs:
# - the extractCSVdata.sh option arguments -c, -fl, -fr and -so are applied
#   prior to any computations by applying the formulas
# - if a field value is empty, then NO formula is applied (no computation done)
# - FORMULA number ZERO IS RESERVED as a "no formula" placeholder
#
LOCAL_FORMULAS='
# custom calculations
# fix accel bias due to offset with zero calibration (name has offset amount)
   1~ accel offset -0.01 2         ~ o=0.01; rd((curr+o),2);
   2~ accel offset -0.02 2         ~ o=0.02; rd((curr+o),2);
   3~ accel offset -0.03 2         ~ o=0.03; rd((curr+o),2);
   4~ accel offset  0.01 2         ~ o=0.01; rd((curr-o),2);
   5~ accel offset  0.02 2         ~ o=0.02; rd((curr-o),2);
   6~ accel offset  0.03 2         ~ o=0.03; rd((curr-o),2);
#
# speed conversions (name has number of decimals)
  10~ KPH to MPH 1                 ~ rd((curr*0.6213711922),1);
  11~ MPH to KPH 1                 ~ rd((curr*1.609344),1);
#
# temperature conversions (name has number of decimals)
  20~ Celsius to Fahrenheit 1      ~ rd(((curr*1.8)+32),1);
  21~ Fahrenheit to Celsius 1      ~ rd(((curr-32)/1.8),1);
#
# length conversions
  30~ kilometers to miles 2        ~ rd((curr*0.6213711922),2);
  31~ miles to kilometers 2        ~ rd((curr*1.609344),2);
  32~ meters to feet 1             ~ rd((curr*3.280839895),1);
  33~ feet to meters 1             ~ rd((curr*0.3048),1);
  34~ centimeters to inches 1      ~ rd((curr*0.3937007874),1);
  35~ inches to centimeters 1      ~ rd((curr*2.54),1);
#
# pressure conversions (name has number of decimals)
  40~ psi to bar 2                 ~ rd((curr*0.0689475729),2);
  41~ bar to psi 1                 ~ rd((curr*14.503773773),1);
  42~ psi to kPa 1                 ~ rd((curr*6.8947572932),1);
  43~ kPa to psi 2                 ~ rd((curr*0.1450377377),2);
#
# power & torque conversions (name has number of decimals)
  50~ kW to HP 0                   ~ rd(curr/0.745699872,0);
  51~ HP to kW 0                   ~ rd(curr*0.745699872,0);
  52~ Nm to ft-lb 0                ~ rd(curr*0.7375621493,0);
  53~ ft-lb to Nm 0                ~ rd(curr*1.3558179483,0);
#
# date/time and time conversions
# eNum is short for a UNIX-style UTC "seconds or milliseconds from epoch"
# date/time number where epoch is 1970-01-01 00:00:00[.000] -- an eNum is always
# UTC-based so an "eNum local" is changing the UTC eNum value so the eNum
# represents the same date/time if that date/time was "local" (i.e., in the
# timezone represented by the time-zone offset value given by the tzo value
# (.sss is milliseconds and .n is 1 or more decimals)
  60~ eNum to eNum Local                ~ rd((curr-tzo),0);
  61~ eNum w/msec to eNum Local         ~ rd((curr-(tzo*1000)),0);
  62~ eNum to UTC H:MM:SS               ~ scale=0; s=curr%60; m=(curr/60)%60; h=(curr/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s;
  63~ eNum to local H:MM:SS             ~ scale=0; l=(curr-tzo); s=l%60; m=(l/60)%60; h=(l/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s;
  64~ eNum w/msec to UTC H:MM:SS.sss    ~ scale=0; u=curr/1000; d=curr%1000; s=u%60; m=(u/60)%60; h=(u/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s; ".";d;
  65~ eNum w/msec to local H:MM:SS.sss  ~ scale=0; l=(curr-(tzo*1000)); u=l/1000; d=l%1000; s=u%60; m=(u/60)%60; h=(u/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s; ".";d;
  66~ msec to H:MM:SS.sss               ~ scale=0; u=curr/1000; d=curr%1000; s=u%60; m=(u/60)%60; h=(u/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s; ".";d;
  67~ M.n to H:MM:SS.sss                ~ scale=0; l=rd((curr*60000),0); u=l/1000; d=l%1000; s=u%60; m=(u/60)%60; h=(u/3600)%24; if(h<10)" "; h;":"; if(m<10)"0"; m;":"; if(s<10)"0"; s; ".";d;
#
# built-in functions to convert an epoch number to a date/time in
# YYYY-MM-DD HH:MM:SS[.sss] format (more information in comments above)
  70~ eNum to UTC date/time             ~ eNumToUTCdateTime();
  71~ eNum to local date/time           ~ eNumToLocalDateTime();
  72~ eNum to UTC date/time with msec   ~ eNumToUTCdateTimeMsec();
  73~ eNum to local date/time with msec ~ eNumToLocalDateTimeMsec();
#
# built-in functions to convert a date/time to an epoch number:
# ---
# the supported date/time input formats are (default is format 1):
# 1 YMD HH:MM:SS : e.g., Y-M-D H:M:S or YY/MM/DDTHH:MM:SS.sss
# 2 DMY HH:MM:SS : e.g., D-M-Y H:M:S[.sss] or DD/MMM/YY HH:MM:SS
# 3 MDY HH:MM:SS : e.g., M-D-Y H:M:S[.sss] or MMM/DD/YY HH:MM:SS
#
# - where: Y = year, M = month, D = day and
#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
# - M, D, H, M and S can be 1 or 2 digits
# - M can also be the month-name abbreviations consisting of the first 3
#   characters of the English month name, Jan-Dec (not case sensitive)
# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
#   is added to Y (e.g., Y=123 ==> 2123, Y=10 ==> 2010) ...  2-digit years,
#   because the century is removed, are valid only for the current century
# - date separators can be either - or /
# - time is HH:MM:SS[.sss]  (where [.sss] means "with or without msec")
# - the date/time separator can be either a space or T (not case sensitive)
  74~ UTC date/time YMD to eNum         ~ UTCdateTimeYMDtoEnum();
  75~ local date/time YMD to eNum       ~ localDateTimeYMDtoEnum();
  76~ UTC date/time DMY to eNum         ~ UTCdateTimeDMYtoEnum();
  77~ local date/time DMY to eNum       ~ localDateTimeDMYtoEnum();
  78~ UTC date/time MDY to eNum         ~ UTCdateTimeMDYtoEnum();
  79~ local date/time MDY to eNum       ~ localDateTimeMDYtoEnum();
# 
# built-in function to transpose a date/time "timestamp" from one format to a
# different format:
# ---
# the supported date/time input/from formats are (default is format 1):
# 1 YMD HH:MM:SS : e.g., Y-M-D H:M:S or YY/MM/DDTHH:MM:SS.sss
# 2 DMY HH:MM:SS : e.g., D-M-Y H:M:S[.sss] or DD/MMM/YY HH:MM:SS
# 3 MDY HH:MM:SS : e.g., M-D-Y H:M:S[.sss] or MMM/DD/YY HH:MM:SS
#
# - where: Y = year, M = month, D = day and
#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
# - M, D, H, M and S can be 1 or 2 digits
# - M can also be the month-name abbreviations consisting of the first 3
#   characters of the English month name, Jan-Dec (not case sensitive)
# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
#   is added to Y (e.g., Y=123 ==> 2123, Y=10 ==> 2010)
# - date separators can be either - or /
# - time is HH:MM:SS[.sss]  (where [.sss] means "with or without msec")
# - date/time separator can be either a space or T (not case sensitive)
#
# the supported date/time output/to formats are (default is format 1):
# YMD formats
#  1 YYYY-MM-DD HH:MM:SS.sss
#  2 YYYY-mmm-DD HH:MM:SS.sss
#  3 YY-MM-DD HH:MM:SS.sss
#  4 YY-mmm-DD HH:MM:SS.sss
# 
#  5 YYYY/MM/DD HH:MM:SS.sss
#  6 YYYY/mmm/DD HH:MM:SS.sss
#  7 YY/MM/DD HH:MM:SS.sss
#  8 YY/mmm/DD HH:MM:SS.sss
# 
# DMY formats
#  9 DD-MM-YYYY HH:MM:SS.sss
# 10 DD-mmm-YYYY HH:MM:SS.sss
# 11 DD-MM-YY HH:MM:SS.sss
# 12 DD-mmm-YY HH:MM:SS.sss
# 
# 13 DD/MM/YYYY HH:MM:SS.sss
# 14 DD/mmm/YYYY HH:MM:SS.sss
# 15 DD/MM/YY HH:MM:SS.sss
# 16 DD/mmm/YY HH:MM:SS.sss
# 
# MDY formats
# 17 MM-DD-YYYY HH:MM:SS.sss
# 18 mmm-DD-YYYY HH:MM:SS.sss
# 19 MM-DD-YY HH:MM:SS.sss
# 20 mmm-DD-YY HH:MM:SS.sss
# 
# 21 MM/DD/YYYY HH:MM:SS.sss
# 22 mmm/DD/YYYY HH:MM:SS.sss
# 23 MM/DD/YY HH:MM:SS.sss
# 24 mmm/DD/YY HH:MM:SS.sss
# 
# - where: Y = year, M = month, D = day and
#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
# - M and D  1 or 2 digits
# - H, M and S are always 2 digits and the optional .sss is always 3 digits
# - mmm is a month-name abbreviation consisting of the first 3 characters
#   of the English name for the month, Jan-Dec (not case sensitive)
# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
#   is added to Y (e.g., Y=123 ==> 2123, Y=10 ==> 2010)
# - date separators can be either - or /
# - time is HH:MM:SS.sss  (where .sss means "with or without msec")
# - date/time separator can be either a space or T (not case sensitive)
#
# the built-in function has parameters, making a large number of conversions
# possible:
# tDT(inputFormat, outputFormat, includeMsec, useShortForm)
# where:
# inputFormat is 1, 2 or 3 (as indicated above)
# outputFormat is 1 through 24 (as indicated above)
# includeMsec is 0 to ignore milliseconds and 1 to include milliseconds
# useShortForm is 0 to include leading zeros and 1 to eliminate leading zeros
#                 in day, month and year numbers
# a few examples are provided, thousands more are possible ...
#
# normalize date/time to (sortable) YYYY-MM-DD HH:MM:SS.sss
  80~ YMD H:M:S[.sss] --> YYYY-MM-DD HH:MM:SS.sss   ~ tDT(1,1,1,0)
  81~ DMY H:M:S[.sss] --> YYYY-MM-DD HH:MM:SS.sss   ~ tDT(2,1,1,0)
  82~ MDY H:M:S[.sss] --> YYYY-MM-DD HH:MM:SS.sss   ~ tDT(3,1,1,0)
#
# coordinate conversions
  90~ lat: decimal degrees to minutes   ~ curr*60.0;
  91~ long: decimal degrees to minutes  ~ curr*-60.0;
#
# angular conversions (name has number of decimals)
  95~ degrees to radians 2         ~ rd((curr*0.01745329252),2);
  96~ radians to degrees 2         ~ rd((curr*57.295779513),2);
#
# simple exponential data smoothing (name has alpha and number of decimals)
# the data-smoothing alpha is between 0 and 1 where alpha=1 yields no smoothing
 100~ data smoothing 0.75 1        ~ a=0.75; b4(rd(((a*curr)+((1-a)*prior)),1));
 101~ data smoothing 0.75 2        ~ a=0.75; b4(rd(((a*curr)+((1-a)*prior)),2));
 102~ data smoothing 0.75 3        ~ a=0.75; b4(rd(((a*curr)+((1-a)*prior)),3));
 103~ data smoothing 0.85 1        ~ a=0.85; b4(rd(((a*curr)+((1-a)*prior)),1));
 104~ data smoothing 0.85 2        ~ a=0.85; b4(rd(((a*curr)+((1-a)*prior)),2));
 105~ data smoothing 0.85 3        ~ a=0.85; b4(rd(((a*curr)+((1-a)*prior)),3));
#
# example of capturing field values in a user-defined array value while passing
# through the value for the field
 300~ capture/set-field curr 1     ~ sf[1]=curr; rd(curr,1);
 301~ capture/set-field curr 2     ~ sf[2]=curr; rd(curr,1);
# example of using above user-defined values to generate a new field value that
# is the average of the 2 fields to which the previous formulas were applied
 310~ average two fields 1         ~ rd(((sf[1]+sf[2])/2),1);
#
# "do nothing" value pass-throughs for testing formula output
 500~ value pass-through 0         ~ curr;
 501~ value pass-through 1         ~ curr+0;
 502~ value pass-through 2         ~ curr+(prev*0);
 503~ value pass-through 3         ~ b4(curr+(prior*0));
#
# the built-in/provided rounding function is defined as:
# define rd(n,decs){auto d,f,x,z;d=decs;x=0;z=scale;scale=d+1;if(n<0){x=1;n=-(n)};f=n+(5/10^(d+1));scale=d;f=(f*(10^d))/(10^d);if(x>0){f=-(f)};scale=z;return(f)};
'
#-------------------------------------------------------------------------------
#
# this can optionally be used to define the location of a file containing the
# formula entries ... if non-empty, it overrides LOCAL_FORMULAS, above, but is
# itself overridden by a "-fp <formulas pathname>" script argument, so:
# the hierarchy, with highest priority first, is:
# - filename via -fnl argument
# - filename set in FORMULAS_FILE variable
# - formulas defined in LOCAL_FORMULAS variable
#
#FORMULAS_FILE=''
FORMULAS_FILE=`/usr/bin/dirname "$0"`'/CSVformulas.txt'
#
#-------------------------------------------------------------------------------
#
# define the time-zone offset from UTC time -- if TIME_ZONE_OFFSET is the empty
# string ('' or ""), use the system's time-zone offset value
# NOTE that the "-tz <[-][H]HMM>" argument overrides all other values
#TIME_ZONE_OFFSET=''      # use OS-supplied timezone
#TIME_ZONE_OFFSET='-700'  # Pacific Daylight Time
TIME_ZONE_OFFSET='-800'  # Pacific Standard Time
#
#+++++ END: USER CONFIGURATION AREA ++++++++++++++++++++++++++++++++++++++++++++

# *** IMPORTANT : don't start other variable names with CSV_FIELDS_

#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#+++ ADDITIONAL/CUSTOM PROCESSING TO ENSURE CORRECT HEADERS +++++++++++++++++
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# the strategy is to accumulate a collection of stream-filtering commands
# that will be applied to the source CSV file to create a temporary work file

filterCMD=''  # the "global" filter command variable that accumulates filters
CHANGE_SEPARATOR=''  # "global" indicates sep-char to change into comma --> CSV

# this section is wrapped as a function so the CSV_FILE_PATHNAME can later be
# passed "back"/in as $1
function getFiltCmd() {
	# regardless of the source/creator/type/format of the CSV file, when applied,
	# the filters in this section must produce a file that contains ONLY:
	# - record 1 that has the CSV field-header record which identifies all fields
	# - subsequent records contain ONLY field-data entries (empty fields allowed)
	# - commas are always and only used to separate fields: i.e., it's simple CSV

	# NOTE: the last entry in any added filter should end with " | \"

	# if required, add the separator translation (first, ensure it's only 1 char)
	if test "$CHANGE_SEPARATOR" != ''
	then
		CHANGE_SEPARATOR=`
						echo "$CHANGE_SEPARATOR" | /usr/bin/sed -e 's:^\(\*.\).*:\1:'`
		filterCMD="$filterCMD"'
			# translate the field-separators into commas
			/usr/bin/tr "'"\\$CHANGE_SEPARATOR"'" "," | \'
	fi
	# identify some CSV files produced by AIM
	AIM_ID=`/usr/bin/fgrep 'AIM CSV File' "$1"`
	#echo "AIM_ID = '$AIM_ID'"

	# identify some CSV files produced by Harry's Laptimer
	LAPTIMER_ID=`/usr/bin/fgrep "Harry's GPS LapTimer" "$1"`
	#echo "AIM_ID = '$AIM_ID'"

	# if it's a CSV file produced by AIM, define a filter command that will:
	# - remove records that are neither heading nor data records
	# - remove any end-of-line commas
	if test "$AIM_ID" != ''
	then
		filterCMD="$filterCMD"'
			# remove the AIM ID and info preamble lines
			/usr/bin/sed -e "1,/^[\ \	]*$/ d" | \
			# remove the extra header-info lines
			/usr/bin/sed -e "2,/^[\ \	]*$/ d" | \'
		#echo "(Processing an AIM CSV file)"
	fi
	#echo "\nfilterCMD=$filterCMD"

	# if it's a CSV file produced by Harry's Laptimer, define a filter command
	# that will:
	# - remove the LapTimer identification line
	# - change the "FIXTYPE" field heading to produce a valid simple CSV header
	if test "$LAPTIMER_ID" != ''
	then
		filterCMD="$filterCMD"'
			/usr/bin/sed -e "1 d" | \
			/usr/bin/sed -e "1 s:FIXTYPE\[COMBUSTION,CSC,ELECTRIC,HYBRID\]:FIXTYPE[COMBUSTION/CSC/ELECTRIC/HYBRID]:" \'
		#echo "(Processing an Laptimer CSV file)"
		#echo "\nfilterCMD=$filterCMD"

		# some Laptimer CSV files have an extra field! ... if so, remove it
		LAPTIMER_HEADER_REC=`
			/usr/bin/sed '2 !d' "$1" |           \
				/usr/bin/tr "\t" ' ' |            \
					/usr/bin/sed -e 's/^[ ]*//'    \
									 -e 's/[ ]*$//'    \
									 -e 's/[ ]*,/,/g'  \
									 -e 's/,[ ]*/,/g'  \
									 -e 's/~/-/g'      \
									 -e 's/\?/-/g'     \
									 -e 's/[ ]*,$//' | \
						/usr/bin/sed -e \
		"s:FIXTYPE\[COMBUSTION,CSC,ELECTRIC,HYBRID\]:FIXTYPE[COMBUSTION/CSC/ELECTRIC/HYBRID]:"`
		#echo "\nLAPTIMER_HEADER_REC = $LAPTIMER_HEADER_REC"
		NUM_HEADER_COMMMAS=`/bin/echo -n "$LAPTIMER_HEADER_REC" | \
										/usr/bin/sed -e 's/[^,]//g' | /usr/bin/wc -m | \
											/usr/bin/sed -e 's/^[\ \	]*//'`
		NUM_DATA_COMMMAS=`/usr/bin/sed '3 !d' "$1" | \
										/usr/bin/sed -e 's/[^,]//g' | /usr/bin/wc -m | \
											/usr/bin/sed -e 's/^[\ \	]*//'`
		#echo "\nNUM_HEADER_COMMMAS = $NUM_HEADER_COMMMAS"
		#echo "NUM_DATA_COMMMAS = $NUM_DATA_COMMMAS"

		# if required, remove the extra end-of-data-record field
		# (regardless, close/finish the filter entry)
		if test `/bin/expr $NUM_HEADER_COMMMAS + 1` -eq $NUM_DATA_COMMMAS
		then
			filterCMD="$filterCMD"'
							 -e "2,$ s/,0.00$//" | \'
		else
			filterCMD="$filterCMD"' | \'
		fi
		#echo "\nfilterCMD=$filterCMD"
	fi
	#echo "\nfilterCMD=$filterCMD"

	# DON'T ADD FILTERS AFTER THIS ONE, ADD FILTERS ABOVE
	# - if it's a RaceRender data CSV, remove the RaceRender identification line
	# - if it's a Vicovation dashcam CSV, remove the VICO identification line
	# - if it's a line that begins with a hash (#) character, remove it (comment)
	# - if the header/1st record has an end-of-line comma, remove it
	filterCMD="$filterCMD"'
		/usr/bin/sed -e "/^# RaceRender Data/d" \
						 -e "/^VICO/d" \
						 -e "/^#/d" | \
		/usr/bin/sed -e "1 s/[\ \	]*,$//"'
	#echo "\nfilterCMD=$filterCMD"
}
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#+++ END OF ADDITIONAL/CUSTOM PROCESSING ++++++++++++++++++++++++++++++++++++
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

# help prevent "sed: RE error: illegal byte sequence" errors (OS X 10.8!)
LC_CTYPE=C

# "macros" -- well, conceptually, at least
bON=`/usr/bin/tput bold`  # set terminal to bold
bOFF=`/usr/bin/tput rmso`  # set terminal to non-bold
BEL=`/usr/bin/tput bel`  # ring terminal's bell or flash, if set to visual bell

# trap interrupts and cancel processing
trap 'doInterrupt; exit 255' TERM INT
#-------------------------------------------------------------------------------
# handle some interrupts
function doInterrupt()
{
	/bin/rm -f "$WORK_FILE_PATHNAME"
	echo "\nProcessing interrupted -- quitting${BEL}\n"
}
#-------------------------------------------------------------------------------

# option defaults
JOIN_FIELDS=0
TO_BE_JOINED_FIELD_NUMS=''
#
SHOW_HELP='0'
SHOW_INFO='0'
SHOW_INFO_AND_QUIT='0'
SHOW_FORMULAS_AND_QUIT='0'
FILL_EMPTY_FIELDS='0'
DO_TRIM='1'  # default is to trim whitespace before/after incoming field values
REMOVE_QUOTES='0'
# CHANGE_SEPARATOR=''  # this is defined above
RESULTS_FILE_NAME_SUFFIX=''
#
SELECTED_SOURCE_CSV_FIELD_NUMS=''
HAVE_fl_ARG='0'
REQUIRED_FIELDS_FLAGS='-1'  # -1 indicates that no required-field flags supplied
HAVE_fr_ARG='0'
#
SORT_ORDER_LIST='-1'    # -1 indicates that no sort-order list was supplied
HAVE_so_ARG='0'
REMOVE_DUPLICATES=''    # default is to keep records with duplicate sort keys
SORT_CASE_OPTION=' -f'  # default is to do a lower-case/case-insensitive sort
NUMERIC_COLLATION=' -n' # default is string number collation rules
DO_MONTH_SORT=''        # don't consider JAN-DEC as months 01-12, when sorting
SORT_DIRECTION=''       # default is ascending sort order
#
GENERATE_STATS='0'
SEPARATE_STATS_FILE='0'
PAGE_WIDTH=80
FIELD_WIDTH=22
#
THE_CSV_FIELDS_SPEC_ID=''  # NOTE: can't be named CSV_FIELDS_SPEC_ID
FIELDS_SPEC_PATHNAME=''
DO_SPACE_FILLING='1'   # default: try to space-fill empty fields to format-width
#
FORMULA_NUM_LIST='-1'  # -1 indicates that no formula number list was supplied
HAVE_fnl_ARG='0'
FORMULAS_PATHNAME=''
#TIME_ZONE_OFFSET=''  # this is defined above
#
OPEN_CONFIG_FILES='0'
SHOW_SELECTED_FIELDS='0'
RESULTS_AS_FORMULAS='0'
SHOW_WORKFILE='0'

FIELD_FORMATS_ARE_USED='0'
SORTING_IS_REQUIRED='0'
FORMULAS_ARE_USED='0'
CURR_YR_NUM=`/bin/date '+%Y'`

# quick 'n dirty argument/options parsing --------------------------------------
while true
do
	if test "$1" = '-j'
	then
		JOIN_FIELDS='1'
		shift
		continue
	fi

	if test "$1" = '-h'
	then
		SHOW_HELP='1'
		shift
		continue
	fi

	if test "$1" = '-hu'
	then
		/usr/bin/sed -e '/START\: USER CONFIG/,/END\: USER CONFIG/ !d' "$0" | \
			/usr/bin/sed -e 's/^\([A-Z][A-Za-z0-9_]*\)=/'"${bON}"'\1'"${bOFF}"'=/'
		exit
	fi

	if test "$1" = '-i'
	then
		SHOW_INFO='1'
		shift
		continue
	fi

	if test "$1" = '-iq'
	then
		SHOW_INFO='1'
		SHOW_INFO_AND_QUIT='1'
		shift
		continue
	fi

	if test "$1" = '-fq'
	then
		SHOW_FORMULAS_AND_QUIT='1'
		shift
		continue
	fi

	if test "$1" = '-c'
	then
		FILL_EMPTY_FIELDS='1'
		shift
		continue
	fi

	if test "$1" = '-nt'
	then
		DO_TRIM='0'
		shift
		continue
	fi

	if test "$1" = '-rq'
	then
		REMOVE_QUOTES='1'
		shift
		continue
	fi

	if test "$1" = '-rc'
	then
		shift
		CHANGE_SEPARATOR="$1"
		shift
		continue
	fi

	if test "$1" = '-rs'
	then
		shift
		RESULTS_FILE_NAME_SUFFIX="$1"
		shift
		continue
	fi

	if test "$1" = '-fl'
	then
		shift
		SELECTED_SOURCE_CSV_FIELD_NUMS="$1"
		HAVE_fl_ARG='1'
		shift
		continue
	fi

	if test "$1" = '-fr'
	then
		shift
		REQUIRED_FIELDS_FLAGS="$1"
		HAVE_fr_ARG='1'
		shift
		continue
	fi

	if test "$1" = '-so'
	then
		shift
		SORT_ORDER_LIST="$1"
		HAVE_so_ARG='1'
		shift
		continue
	fi

	if test "$1" = '-rd'
	then
		shift
		REMOVE_DUPLICATES=' -u' # remove records w/duplicate sort-key field values
		continue
	fi

	if test "$1" = '-cs'
	then
		shift
		SORT_CASE_OPTION=''  # do case-sensitive sorting
		continue
	fi

	if test "$1" = '-sa'
	then
		shift
		NUMERIC_COLLATION='' # don't use numeric sorting, use alphabetic collation
		continue
	fi

	if test "$1" = '-sm'
	then
		shift
		DO_MONTH_SORT=' -M'  # consider JAN-DEC as months 01-12 when sorting
		continue
	fi

	if test "$1" = '-sd'
	then
		shift
		SORT_DIRECTION=' -r'
		continue
	fi

	if test "$1" = '-m'
	then
		GENERATE_STATS='1'
		shift
		continue
	fi

	if test "$1" = '-ms'
	then
		GENERATE_STATS='1'
		SEPARATE_STATS_FILE='1'
		shift
		continue
	fi

	if test "$1" = '-pw'
	then
		shift
		PAGE_WIDTH=$1

		if test \( $PAGE_WIDTH -lt 60 \) -a \( "$SHOW_HELP" != '1' \)
		then
			echo "\nWarning: the specified page width ($PAGE_WIDTH) was invalid and was set to 80"
			PAGE_WIDTH=80
		fi

		shift
		continue
	fi

	if test "$1" = '-fw'
	then
		shift
		FIELD_WIDTH=$1

		if test \( $FIELD_WIDTH -lt 6 \) -o \( $FIELD_WIDTH -gt 100 \) -a \
				  \( "$SHOW_HELP" != '1' \)
		then
			echo "\nWarning: the specified field width ($FIELD_WIDTH) was invalid and was set to 22"
			FIELD_WIDTH=22
		fi

		shift
		continue
	fi

	if test "$1" = '-fi'
	then
		shift
		THE_CSV_FIELDS_SPEC_ID=`
					echo "$1" | /usr/bin/sed -e 's/^[\ \	]*//' -e 's/[\ \	]*$//'`
		shift
		continue
	fi

	if test "$1" = '-fsp'
	then
		shift
		FIELDS_SPEC_PATHNAME="$1"
		shift
		continue
	fi

	if test "$1" = '-nf'
	then
		shift
		DO_SPACE_FILLING='0'
		continue
	fi

	if test "$1" = '-fnl'
	then
		shift
		FORMULA_NUM_LIST="$1"
		HAVE_fnl_ARG='1'
		shift
		continue
	fi

	if test "$1" = '-fp'
	then
		shift
		FORMULAS_PATHNAME="$1"
		shift
		continue
	fi

	if test "$1" = '-tz'
	then
		shift
		TIME_ZONE_OFFSET="$1"
		shift
		continue
	fi

	if test "$1" = '-o'
	then
		OPEN_CONFIG_FILES='1'
		shift
		continue
	fi

	if test "$1" = '-ss'
	then
		SHOW_SELECTED_FIELDS='1'
		shift
		continue
	fi

	if test "$1" = '-sf'
	then
		RESULTS_AS_FORMULAS='1'
		shift
		continue
	fi

	if test "$1" = '-w'
	then
		SHOW_WORKFILE='1'
		shift
		continue
	fi

	# if just getting help or formulas, no need to get CSV-file pathname
	if test \( "$SHOW_HELP" = '1' \) -o \( "$SHOW_FORMULAS_AND_QUIT" = '1' \) -o\
			  \( "$OPEN_CONFIG_FILES" = '1' \)
	then
		break
	fi

	# if required, get the full pathname for a file that contains record entries
	# in CSV format where the first such record is a field-header record.
	if test "$1" = ''
	then
		echo '
This script reads a file in CSV format, determines the fields (assuming it can
find the field-name heading) and creates a new file that has records which
contain only the selected fields (with various selection/processing options)
-----
Enter the full pathname for the CSV file (or nothing to quit): '
		read CSV_FILE_PATHNAME

		if test "$CSV_FILE_PATHNAME" = ''
		then
			exit
		elif test ! -f "$CSV_FILE_PATHNAME"
		then
			echo
			echo 'ERROR: The CSV file'
			echo "$CSV_FILE_PATHNAME"
			echo "could not be found${BEL}"
			echo
			CSV_FILE_PATHNAME=''
			continue
		fi
	else
		CSV_FILE_PATHNAME="$1"
		shift

		if test "$1" != ''  # have another parameter so is arg is likely bad
		then
			echo "\nERROR: The argument '$CSV_FILE_PATHNAME' appears to be invalid ... ignoring${BEL}"
			CSV_FILE_PATHNAME=''
			continue
		elif test ! -f "$CSV_FILE_PATHNAME"
		then
			echo
			echo 'ERROR: The CSV file'
			echo "$CSV_FILE_PATHNAME"
			echo "could not be found${BEL}"
			echo
			CSV_FILE_PATHNAME=''
			continue
		fi
	fi

	break
done

if test "$JOIN_FIELDS" = '1'
then
	echo "\nNOTE that all options other than -i, -rs, -fl, and -w are ignored when joining fields"

	SHOW_HELP='0'
	SHOW_INFO_AND_QUIT='0'
	SHOW_FORMULAS_AND_QUIT='0'
	FILL_EMPTY_FIELDS='0'
	DO_TRIM='0'
	REMOVE_QUOTES='0'
	CHANGE_SEPARATOR=''
	#
	if test "$RESULTS_FILE_NAME_SUFFIX" = ''
	then
		RESULTS_FILE_NAME_SUFFIX='-joined'
		echo '     and the <results suffix> will be "-joined" instead of "-selected"'
	fi
	#
	REQUIRED_FIELDS_FLAGS='-1'
	HAVE_fr_ARG='0'
	#
	SORT_ORDER_LIST='-1'
	HAVE_so_ARG='0'
	#
	GENERATE_STATS='0'
	SEPARATE_STATS_FILE='0'
	#
	THE_CSV_FIELDS_SPEC_ID=''
	DO_SPACE_FILLING='0'
	#
	FORMULA_NUM_LIST='-1'
	HAVE_fnl_ARG='0'
	#
	OPEN_CONFIG_FILES='0'
	SHOW_SELECTED_FIELDS='0'
	RESULTS_AS_FORMULAS='0'
fi

if test "$SHOW_HELP" = '1'
then
		echo '
'"${bON}"'Description'"${bOFF}"':
  This script reads a file that contains line-item/record entries in CSV format.
  If it can locate the field-header entry, it then:
  - determines the fields in the source CSV file
  - gets the selection of fields to be included in the results file
  - ignores/reports any entries that do not have the correct number of fields
  - creates a separate results file containing only the specified fields in the
    order specified via the supplied/entered field list
  - optionally sorts the records using a specified collection of sorting fields
  - optionally, "copies forward" previous field values to produce a CSV file
    where all records have no empty fields
  - optionally, ignores record entries that do not have a value for any field
    that is identified via a "required fields" list
  - optionally, formats the data fields in the result file
  - optionally, transposes data by applying formulas to specified fields
  - optionally, transposes CSV header field names to new field names used in the
    results file header record
  - optionally, joins one or more fields into a single field (e.g., date & time)
  - optionally, changes tab/space/other character to be a comma field-separator
  - optionally, produces statistics about the selected records and fields
  - places results in a separate file named:
       <processed file name>-<results suffix>.<processed file suffix>
    where "-rs <results suffix>" argument overrides the default "-selected" value

Synopsis:  (square brackets means "is optional")
'"${bON}"'extractCSVdata.sh'"${bOFF}"' [i] -j [-fl <field list>] [-w] [<CSV pathname>]
-- OR --
'"${bON}"'extractCSVdata.sh'"${bOFF}"'
  [[-h] [-hu] [-i] [-iq] [-fq] [-c] [-nt] [-rq] [-rc <separator>] [-rs <results suffix>]
   [-fl <field list>] [-fr <field-required list>]
   [-so <sort-order list>] [-rd] [-cs] [-sa] [-sm] [-sd]
   [[-m] [-ms] [-pw <page width>] [-fw <field width]]
   [[-fi <fields ID>] [-fsp <fields spec pathname>] [-nf]]
   [-fnl <formula-number list>] [-fp <formulas pathname>] [-tz <[-][H]HMM>]
   [-o] [-ss] [-sf] [-w]] [<CSV pathname>]

the following options fit with all other options:
  [-h] [-hu] [-i] [-iq] [-fq] [-c] [-nt] [-rq] [-rc <separator>] [-rs <results suffix>]
  [[-m] [[-ms] [-pw <page width>] [-fw <field width]]]
  [-o] [-ss] [-sf] [-w]] [<CSV pathname>]

the following option sets are compatible (incompatible options cause overrides):
  [-fl <field list>] [-fr <field-required list>]
  [-so <sort-order list>] [-rd] [-cs] [-sa] [-sm] [-sd]
  [-fnl <formula-number list>] [-fp <formulas pathname>] [-tz <[-][H]HMM>]
---
  [[-fi <fields ID>] [-fsp <fields spec pathname>] [-nf]]
  [-fp <formulas pathname>] [-tz <[-][H]HMM>]

Where:  (<CSV pathname> must appear last, if provided)
'"${bON}"'-j'"${bOFF}"' : joins specified fields -- i.e., creates a result file with records that
  contain all source CSV fields where, for each field in the field list, the
  after-field comma separator:
  - is replaced by a bar (|) character in the header record
  - is replaced by a space ( ) caracter in the data records
  (i.e., header fields are "bar joined" and data fields are "space joined")

'"${bON}"'-h'"${bOFF}"' : shows this help message, then quits
'"${bON}"'-hu'"${bOFF}"' : shows this script'"'"'s internal '"${bON}"'USER CONFIGURATION AREA'"${bOFF}"' content, then quits
'"${bON}"'-i'"${bOFF}"' : shows information about the current run (documents the run configuration)
'"${bON}"'-iq'"${bOFF}"' : shows information about current run, then quits (useful when configuring)
'"${bON}"'-fq'"${bOFF}"' : shows information about available formulas, then quits (also see -fp)
'"${bON}"'-c'"${bOFF}"' : causes copy-forward processing where previous values fill empty-field values:
  - creates a file with zero empty fields (sometimes a CSV requirement)
  - removes initial records that are missing one or more field values, prior to
    accumulating a value for every field (reports number of records removed)
  - records missing a field still pariticipate in field-filling search
'"${bON}"'-nt'"${bOFF}"' : cancels the removal of whitespace before/after data-field values (no-trim)
  (a whitespace-only field is not considered empty and is not numeric-formatted)
'"${bON}"'-rq'"${bOFF}"' : removes all double-quote characters (including from the header record)
'"${bON}"'-rc <separator>'"${bOFF}"' : replaces all occurrences of the source file'"'"'s field <separator>
  character with a comma to create a CSV file (e.g., convert tab/space-separated)
'"${bON}"'-rs <results suffix>'"${bOFF}"' : overrides the "-selected" part of the results-file name --
  i.e., so results file will be named
          <processed file name><results suffix>.<processed file suffix>
        [only alphabetic, numeric, hyphen (-) and/or underscore (_) characters]

'"${bON}"'-fl <field list>'"${bOFF}"' : provides a comma-separated list of field numbers to be
  included in the results file:
  - list-entry numbers correspond to the left-to-right source CSV field positions
  - "all" can be used to include all available source CSV fields
  - if missing, the list of available fields will be shown so the field list can
    be entered interactively
'"${bON}"'-fr <field-required list>'"${bOFF}"' : provides a comma-separated list of "is required" flags:
  - the list entries correspond to the left-to-right selected CSV field positions
  - a 1 entry indicates that the corresponding source CSV field MUST have a
    value or that entire source record will be eliminated from the results file
  - a 0 or empty entry means the source CSV field will be included in results file
  - if the field-required list is the empty string ('"''"' or ""), the list of fields
    will be shown so the required-fields flags can be entered interactively

'"${bON}"'-so <sort-order list>'"${bOFF}"' : provides a comma-separated list of field numbers that
  yields the ordered fields by which source CSV records will be sorted before
  being processed (e.g., to ensure that time-ordered data is properly ordered):
  - the list entries correspond to the left-to-right result-file field positions
  - a 0 value or empty entry means "do not sort by this field"
  - a non-zero/non-empty value indicates the sort priority for that field position
    where lower numbers have higher priority --e.g.:
    "0,5,0,3" would sort first by field 4 (priority 3) then by field 2 (priority 5)
  - if the sort-order list is the empty string ('"''"' or ""), the list of fields will
    be shown so the sort-order list can be entered interactively
'"${bON}"'-rd'"${bOFF}"' : keeps only the first source record when subsequent records have duplicate
  (identical) values for the sorted-by fields (only considers sorting fields!)
'"${bON}"'-cs'"${bOFF}"' : sorts using case-sensitive alpabetic collating rules
  when comparing field values, instead of case-insensitive collating rules
'"${bON}"'-sa'"${bOFF}"' : sorts using alpabetic collating rules when comparing
  field values, instead of using the numeric (string) collating rules
'"${bON}"'-sm'"${bOFF}"' : sorts the 3-character sequences JAN-DEC as months 01-12
'"${bON}"'-sd'"${bOFF}"' : sorts in descending order, instead of default ascending order

'"${bON}"'-m'"${bOFF}"' : generates statistics/meta info (min, max, max-min, average, # of changes)
'"${bON}"'-ms'"${bOFF}"' : generates statistics info in a separate "-<results suffix>-stats" file
  file, instead of at the end of the "-<results suffix>" file [overrides -m]
'"${bON}"'-pw <page-width>'"${bOFF}"' : sets the width of the page, in characters, into which the
  statistics info will be formatted (default = 80, minimum = 60) [requires -m/-ms]
'"${bON}"'-fw <field width>'"${bOFF}"' : sets the minimum width of the fields, in characters, into
  which the statistics info will be formatted ... most useful to allow room for
  longer/non-truncated headings (min = 6, default = 22, max = 100) [requires -m/-ms]

'"${bON}"'-fi <fields ID>'"${bOFF}"' : provides an identifier to select a "CSV_FIELDS" specification
  that will be used to configure the CSV field-extraction process -- this allows
  multiple "CSV_FIELDS_<fields ID>" to be defined, selected & used:
  e.g., different CSV_FIELDS specifications can be setup for different types of
        CSV files and/or variants of the same CSV file types ('"${bON}"'see NOTEs below'"${bOFF}"')
  (characters not alphabetic, alphanumeric and/or underscore are ignored/remmoved)
  [overrides other related option arguments; -fl, -fr, -so and -fnl]
'"${bON}"'-fsp <fields spec pathname>'"${bOFF}"' : provides the pathname to a file that contains the
  CSV field-extraction (CSV_FIELDS) specifications that are to be used and, if
  present, overrides any specifications provided via CSV_FIELDS_<fields ID> & the
  THE_CSV_FIELDS_SPEC_FILE variable (both are defined inside extractCSVdata.sh)
'"${bON}"'-nf'"${bOFF}"' : cancels attempts to fill empty fields with a format-width number of spaces
  (no effect with format %s, space-filling can cause problems in some scenarios)

'"${bON}"'-fnl <formula-number list>'"${bOFF}"' : provides a comma-separated list of formula numbers:
  - the list entries correspond to the left-to-right result-file field positions
  - a 0 value means "do not apply any formula to transform this field"
  - a non-zero/non-empty value indicates the number of the formula that will be
    applied to transform the source-field value into the result-field value
  - if the formula list is the empty string ('"''"' or ""), the available formulas
    will be shown so the list can be entered interactively ('"${bON}"'see NOTEs below'"${bOFF}"')
  - double-quotes (") are removed from data records when formulas are being used
'"${bON}"'-fp <formulas pathname>'"${bOFF}"' : provides the pathname to the formulas file and, if
  present, overrides any formulas provided via LOCAL_FORMULAS and the FORMULAS_FILE
  variable, both of which are defined inside extractCSVdata.sh
'"${bON}"'-tz <[-][H]HMM>'"${bOFF}"' : provides a time-zone offset from UTC that will override
  any current script-defined or system-supplied time-zone offset -- sets the value
  for the built-in "tzo" variable that can be used in formulas (must be a string)

'"${bON}"'-o'"${bOFF}"' : opens, for editing, the file(s) containing the fields IDs and the formulas,
  then quits -- uses TEXT_EDITOR or TextEdit -- see the USER CONFIGURATION AREA
'"${bON}"'-ss'"${bOFF}"' : shows the selected/sorted records/fields prior to the major field processing
'"${bON}"'-sf'"${bOFF}"' : shows the fields/records/formulas in the form that is fed to the bc utility
'"${bON}"'-w'"${bOFF}"' : shows temporary work file (useful when customizing filter for new file type)

'"${bON}"'<CSV pathname>'"${bOFF}"' : provides the name of the CSV file to be processed -- if missing,
  a file pathname will be interactively requested

'"${bON}"'NOTEs'"${bOFF}"':
  - extensive user-configuration is allowed by setting values in the
    extractCSVdata.sh script ... see the section: "'"${bON}"'USER CONFIGURATION AREA'"${bOFF}"'"
  - this script assumes a CSV format where commas always/only separate fields
  - for CSV files that contain entries other than header and data records, some
    additional processing may be required ... in the extractCSVdata.sh script,
    see the section: ADDITIONAL/CUSTOM PROCESSING TO ENSURE CORRECT HEADERS
  - the overall order of processing is:
    * determine all input conditions from arguments and interactive input
    * get the header record
    * create work file containing only header & data records (separator changed)
    * determine the fields that are to be extracted (or joined)
    * determine the conditions of fields extraction (e.g., transposing, sorting)
    * if requested, show the selections and conditions for the run
    * perform the fields extraction (or joining):
      + extract the required fields, ignoring records with missing/extra fields
      + do any specified sorting (or joining) -- uses the source/original fields
      + do the required-fields selection (part of major field processing)
      + do the field filling and apply formulas (part of major field processing)
      + if applicable, do any field formatting
    * if requested, generate the statistics
      
The currently defined <fields ID> values are:'
fi

# if just showing help or the available formulas, much stuff can be skipped --
# it's done this way so we can show the available <field suffix> values when
# showing the help and the formulas when just showing the formulas
if test \( "$SHOW_HELP" != '1' \) -a \( "$SHOW_FORMULAS_AND_QUIT" != '1' \) -a \
		  \( "$OPEN_CONFIG_FILES" != '1' \)
then
	echo "\nGathering info (creating work file) ..."

	# result/work filename and file stuff ---------------------------------------
	# determine the name extension for the generated/results file(s)
	if test "$RESULTS_FILE_NAME_SUFFIX" = ''
	then
		RESULTS_FILE_NAME_SUFFIX='-selected'
	elif test \( "$RESULTS_FILE_NAME_SUFFIX" != '' \) -a \
				 \( "`echo $RESULTS_FILE_NAME_SUFFIX | \
									/usr/bin/sed -e 's/[\ a-zA-Z0-9\_\-]//g'`" != '' \)
	then
		echo "\nERROR: the '<results suffix>' ($RESULTS_FILE_NAME_SUFFIX)"
		echo "       must only have alphabetic, alphanumeric, hyphen (-), underscore (_)"
		echo "       and/or space characters ... quitting${BEL}"
		exit 1
	fi

	DIR=`/usr/bin/dirname "$CSV_FILE_PATHNAME"`
	FILE_NAME=`/usr/bin/basename "$CSV_FILE_PATHNAME" | \
														/usr/bin/sed -e 's/^\(.*\)\..*$/\1/'`
	FILE_SUFFIX=`/usr/bin/basename "$CSV_FILE_PATHNAME" | \
														/usr/bin/sed -e 's/^.*\.\(.*\)$/\1/'`
	WORK_FILE_PATHNAME="$DIR/${FILE_NAME}-work.$FILE_SUFFIX"

	# create an empty work file
	/bin/echo -n > "$WORK_FILE_PATHNAME"  # create an empty work file

	# get the command to do the initial filtering of the source CSV file (defined
	# earlier, as a function, for user accessibility)
	getFiltCmd "$CSV_FILE_PATHNAME"
	#echo "\nfilterCMD=$filterCMD"

	# create a work copy of the CSV file that has:
	# - UNIX-style (newline only) line/header/record endings
	# - record 1 that has the CSV field-header record which identifies all fields
	# - subsequent records contain ONLY field-data entries (empty fields allowed)
	# - commas are always and only used to separate fields: i.e., it's simple CSV
	/usr/bin/tr -d '\r' < "$CSV_FILE_PATHNAME" | \
		eval "$filterCMD" >> "$WORK_FILE_PATHNAME"

	# quit if the temporary work file is to be shown
	if test \( "$SHOW_WORKFILE" = '1' \) -a \( "$SHOW_INFO_AND_QUIT" != '1' \)
	then
		echo "\nNOTE that the work file"
		echo "$WORK_FILE_PATHNAME"
		echo 'should contain only the source CSV header, as the first line, and source CSV'
		echo 'data records where commas separate, and are only used to separate, the fields'
		echo "\nThe source CSV file has been filtered by the following command(s):"
		/bin/echo -n '/usr/bin/tr -d '"'"'\'""'r'"'"' < "<source CSV file>" | \'
		echo "$filterCMD"
		exit
	fi

	# source CSV fields stuff ------------------------------------------------------
	# get the (now MUST BE first-line-in-file) header record from the CSV file:
	# - change any tab characters to a single space character
	# - remove any whitespace at the start/end of the header record
	# - remove any whitespace adjacent to the comma-separators
	# - change any tilde (~) and question mark (?) characters to dash (-)
	#   characters because the tilde and question mark characters are used as
	#   separator and EOL characters, respectively, in various routines below
	# - remove any comma at the end of the header
	SOURCE_HEADER_REC=`/usr/bin/head -n 1 "$WORK_FILE_PATHNAME" |             \
																		/usr/bin/tr "\t" ' ' | \
															/usr/bin/sed -e 's/^[ ]*//'     \
																			 -e 's/[ ]*$//'     \
																			 -e 's/[ ]*,/,/g'   \
																			 -e 's/,[ ]*/,/g'   \
																			 -e 's/~/-/g'       \
																			 -e 's/\?/-/g'      \
																			 -e 's/[ ]*,$//'`
	#echo "\nSOURCE_HEADER_REC:\n$SOURCE_HEADER_REC"

	# if required, also remove all double-quote characters from the header record
	if test "$REMOVE_QUOTES" = '1'
	then
		SOURCE_HEADER_REC=`echo "$SOURCE_HEADER_REC" | /usr/bin/sed -e 's/"//g'`
	fi
	#echo "\nSOURCE_HEADER_REC:\n$SOURCE_HEADER_REC"

	# create a list of source CSV fields & their left-to-right numeric position
	SOURCE_CSV_FIELD_NUM_NAME_LIST=`
				echo "$SOURCE_HEADER_REC" | \
					/usr/bin/awk -F ',' '
						{ for (i = 1; i <= NF; i++) { printf("%d,%s\n", i, $i) } }'`
	#echo "\nSOURCE_CSV_FIELD_NUM_NAME_LIST\n$SOURCE_CSV_FIELD_NUM_NAME_LIST\n"

	# create a comma-separated list of all the source CSV field numbers
	ALL_ORIG_CSV_FIELD_NUMS=`echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" |         \
										/usr/bin/sed -e 's/^\([0-9][0-9]*\),.*$/\1/' | \
											/usr/bin/tr "\n" ',' |                      \
												/usr/bin/sed -e 's/,$//'`
	#echo "\nALL_ORIG_CSV_FIELD_NUMS = '$ALL_ORIG_CSV_FIELD_NUMS'"

	# create a comma-separated list of all the source CSV field names
	ALL_ORIG_CSV_FIELD_NAMES=`echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
										/usr/bin/sed -e 's/^[0-9][0-9]*,\(.*\)$/\1/' |\
											/usr/bin/tr "\n" ',' | \
												/usr/bin/sed -e 's/,$//'`
	#echo "\nALL_ORIG_CSV_FIELD_NAMES = '$ALL_ORIG_CSV_FIELD_NAMES'"

	# if required, get the list of CSV field numbers for fields to be extracted
	if test \( "$THE_CSV_FIELDS_SPEC_ID" = '' \) -a \
			  \( "$SELECTED_SOURCE_CSV_FIELD_NUMS" = '' \) -a \
			  \( "$SHOW_INFO_AND_QUIT" != '1' \) -a \
			  \( "$SHOW_FORMULAS_AND_QUIT" != '1' \)
	then
		# show the list of source CSV fields
		echo "\nThe complete list of fields in the CSV file ..."
		echo ' Field  Field'
		echo 'Number  Name'
		echo '------  -------------------'
		echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
								/usr/bin/awk -F ',' '{ printf("%5d   %s\n", $1, $2) }'

		# get the fields to be extracted
		echo '-----'

		if test "$JOIN_FIELDS" = '1'
		then
			echo 'Enter a comma-separated list of the CSV field numbers to be joined with the next field:'
			echo '(e.g., 3,6 joins field 3 with field 4 and field 6 with field 7)'
		else
			echo 'Enter "all" or a comma-separated list of the CSV field numbers to be extracted:'
		fi

		read SELECTED_SOURCE_CSV_FIELD_NUMS
	fi

	if test \( "$SELECTED_SOURCE_CSV_FIELD_NUMS" = 'all' \) -a \
			  \( "$JOIN_FIELDS" != '1' \)
	then
		SELECTED_SOURCE_CSV_FIELD_NUMS="$ALL_ORIG_CSV_FIELD_NUMS"
	fi

	if test \( "$SELECTED_SOURCE_CSV_FIELD_NUMS" = '' \) -a \
			  \( "$JOIN_FIELDS" = 1 \)
	then
		echo "\nERROR: no 'fields to be joined' were supplied  -- quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# if applicable, remove the whitespace and end-of-line comma
	SELECTED_SOURCE_CSV_FIELD_NUMS=`echo "$SELECTED_SOURCE_CSV_FIELD_NUMS" | \
												/usr/bin/sed -e 's/[\ \	]//g' -e 's/,$//'`
	#echo "SELECTED_SOURCE_CSV_FIELD_NUMS = '$SELECTED_SOURCE_CSV_FIELD_NUMS'"

	# partially validate the field-number list (ensure only has numbers & commas)
	if test "$JOIN_FIELDS" = 1
	then
		MSG="\nERROR: the 'fields to be joined' list\n$SELECTED_SOURCE_CSV_FIELD_NUMS\nis not valid -- quitting${BEL}"
	else
		MSG="\nERROR: the selected CSV field list\n$SELECTED_SOURCE_CSV_FIELD_NUMS\nis not valid -- quitting${BEL}"
	fi

	if test "`echo $SELECTED_SOURCE_CSV_FIELD_NUMS | \
														/usr/bin/sed -e 's/[0-9\,]//g'`" != ''
	then
		echo "$MSG"
		echo "\nThe available fields are:\n$ALL_ORIG_CSV_FIELD_NUMS"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# if joining fields, set the "to be joined fields" variable and set the
	# selected source CSV fields to be "all fields"
	if test "$JOIN_FIELDS" = '1'
	then
		FULL_TO_BE_JOINED_FIELD_NUMS="$SELECTED_SOURCE_CSV_FIELD_NUMS"
		TO_BE_JOINED_FIELD_NUMS="$SELECTED_SOURCE_CSV_FIELD_NUMS"
		SELECTED_SOURCE_CSV_FIELD_NUMS="$ALL_ORIG_CSV_FIELD_NUMS"
	fi

	# flatten the source list of CSV numbers,names by changing newlines to tildes
	SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE=`
												echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
												/usr/bin/tr "\n" '~' |                   \
												/usr/bin/sed -e 's/~$//'`
	#echo "\nSOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE (unflattened):\n"
	#echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE" | /usr/bin/tr '~' "\n"

	# if required, create a comma-separated list of the selected original/source
	# CSV field names by looking up the name of each source CSV field that
	# (positionally) corresponds to the field number in the supplied list
	if test "$SELECTED_SOURCE_CSV_FIELD_NUMS" = ''
	then
		SELECTED_SOURCE_CSV_FIELD_NAMES=''
	else
		SELECTED_SOURCE_CSV_FIELD_NAMES=`\
			/usr/bin/awk \
			 -v sourceCSVfieldsNumNameList="$SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE" '
				BEGIN {
					# SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain double-quotes
					# NOTE: since SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain
					#       double-quotes, it must be loaded via the -v argument
					#       (unless quotes are escaped)

					# create an array of the source CSV field num/name entries and
					# get the number of source CSV fields
					numSourceCSVfields = split(sourceCSVfieldsNumNameList, \
														sourceCSVfieldNumNameEntry, "~")

					# create an associative array to enable a CSV field name to be
					# looked up (i.e., indexed) by a CSV field number so:
					# CSVfieldName[<CSV field number>] yields the <CSV field name>
					for (i = 1; i <= numSourceCSVfields; i++) {
						split(sourceCSVfieldNumNameEntry[i], CSVfieldNumName, ",")
						CSVfieldName[CSVfieldNumName[1]] = CSVfieldNumName[2]
					}

					# uncomment for debug output
					#for (item in CSVfieldName) {
					#	printf("CSVfieldName[%s] = %s : item = %s\n",
					#											item, CSVfieldName[item], item)
					#}
					#exit

					# load the selected/list of CSV field numbers into awk
					theFieldNums = "'"$SELECTED_SOURCE_CSV_FIELD_NUMS"'"

					# create an array of field numbers & get the number of entries
					numFieldNums = split(theFieldNums, fieldNum, ",")

					# output a comma-separated list containing the name of each
					# source CSV field name that (positionally) corresponds to each
					# field number in the supplied list
					# specification
					isFirstField = 1
					origCSVfieldName = ""

					# get the name of the CSV field that corresponds to the
					# number/position of the selected CSV field number
					for (i = 1; i <= numFieldNums; i++) {
						origCSVfieldName = CSVfieldName[fieldNum[i]]

						if (isFirstField == 1) { isFirstField = 0 }
						else { printf(",") }

						# report invalid field numbers
						if (length(origCSVfieldName) == 0) {
							system("echo >&2 ; echo ERROR: the CSV file has no field at left-to-right position number "fieldNum[i]" >&2")
							origCSVfieldName = "--err--"
						}

						printf("%s", origCSVfieldName)
					}
				}'`
		#echo "\nSELECTED_SOURCE_CSV_FIELD_NAMES = '$SELECTED_SOURCE_CSV_FIELD_NAMES'"

		if test "`echo $SELECTED_SOURCE_CSV_FIELD_NAMES | \
															/usr/bin/fgrep -e '--err--'`" != ''
		then
			echo "\nError(s) encountered ... quitting${BEL}"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		fi
	fi

	# set the list of field names (i.e., the header) for the result file
	RESULT_CSV_FIELD_NAMES="$SELECTED_SOURCE_CSV_FIELD_NAMES"
	#echo "\nRESULT_CSV_FIELD_NAMES = $RESULT_CSV_FIELD_NAMES"

	# required-fields stuff -----------------------------------------------------
	# if required, get the list of required-field flags
	if test \( "$REQUIRED_FIELDS_FLAGS" = '' \) -a \
			  \( "$SHOW_INFO_AND_QUIT" != '1' \)
	then
		# show the list of source CSV fields
		echo "\nThe complete list of fields in the CSV file ..."
		echo ' Field  Field'
		echo 'Number  Name'
		echo '------  -------------------'
		echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
								/usr/bin/awk -F ',' '{ printf("%5d   %s\n", $1, $2) }'

		# get the sort fields
		echo '-----'

		if test "$RESULT_CSV_FIELD_NAMES" != ''
		then
			echo "\nResult-file CSV field names:"
			echo "$RESULT_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
		fi

		if test "$SELECTED_SOURCE_CSV_FIELD_NUMS" != ''
		then
			echo "\nSelected CSV field numbers:"
			echo "$SELECTED_SOURCE_CSV_FIELD_NUMS" | /usr/bin/sed -e 's/,/ , /g'
		fi

		echo "\nEnter a comma-separated list of required-field flags to indicate which of the"
		echo 'source CSV records will be ignored when no value exists for a required field --'
		echo 'a 1 flag indicates the selected field is required to have a non-empty value and'
		echo 'an empty or 0 flag entry indicates an empty-value in the selected field is OK:'
		read REQUIRED_FIELDS_FLAGS
	fi

	if test "$REQUIRED_FIELDS_FLAGS" = '-1'  # no required-field flags supplied
	then
		REQUIRED_FIELDS_FLAGS=''
	fi

	# if applicable, validate the required-fields flag list
	if test "$REQUIRED_FIELDS_FLAGS" != ''
	then
		# remove whitespace and end-of-line comma
		REQUIRED_FIELDS_FLAGS=`echo "$REQUIRED_FIELDS_FLAGS" | \
												/usr/bin/sed -e 's/[\ \	]//g' -e 's/,$//'`
		#echo "REQUIRED_FIELDS_FLAGS = '$REQUIRED_FIELDS_FLAGS'"

		# partially validate the field-number list (ensure only 0, 1 & comma)
		if test "`echo $REQUIRED_FIELDS_FLAGS | \
														/usr/bin/sed -e 's/[01\,]//g'`" != ''
		then
			echo "\nERROR: the required-fields flag list"
			echo "$REQUIRED_FIELDS_FLAGS"
			echo "is not valid -- quitting${BEL}"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		fi
	fi
	#echo "\nREQUIRED_FIELDS_FLAGS = '$REQUIRED_FIELDS_FLAGS'"

	# sorting stuff -------------------------------------------------------------
	# if required, get the list of sort-order numbers
	if test \( "$SORT_ORDER_LIST" = '' \) -a \
			  \( "$SHOW_INFO_AND_QUIT" != '1' \)
	then
		# show the list of source CSV fields
		echo "\nThe complete list of fields in the CSV file ..."
		echo ' Field  Field'
		echo 'Number  Name'
		echo '------  -------------------'
		echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
								/usr/bin/awk -F ',' '{ printf("%5d   %s\n", $1, $2) }'

		# get the sort fields
		echo '-----'

		if test "$RESULT_CSV_FIELD_NAMES" != ''
		then
			echo "\nResult-file CSV field names:"
			echo "$RESULT_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
		fi

		if test "$SELECTED_SOURCE_CSV_FIELD_NUMS" != ''
		then
			echo "\nSelected CSV field numbers:"
			echo "$SELECTED_SOURCE_CSV_FIELD_NUMS" | /usr/bin/sed -e 's/,/ , /g'
		fi

		echo "\nEnter a comma-separated list of sort priorities to indicate the field and"
		echo 'its priority for sorting the extracted source CSV records before processing:'
		echo "(an empty entry or a '0'/zero entry indicates no sorting by a field)"
		read SORT_ORDER_LIST
	fi

	# if required, partially validate the sort-order list
	if test "$SORT_ORDER_LIST" = '-1'  # no sort-order list was supplied
	then
		SORT_ORDER_LIST=''
	else
		# remove whitespace and end-of-line comma
		SORT_ORDER_LIST=`echo "$SORT_ORDER_LIST" | \
												/usr/bin/sed -e 's/[\ \	]//g' -e 's/,$//'`
		#echo "SORT_ORDER_LIST = '$SORT_ORDER_LIST'"

		# partially validate the field-number list (ensure only 0-9 & comma)
		if test "`echo $SORT_ORDER_LIST | /usr/bin/sed -e 's/[0-9\,]//g'`" != ''
		then
			echo "\nERROR: the sort-order list"
			echo "$SORT_ORDER_LIST"
			echo "is not valid -- quitting${BEL}"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		fi
	fi

	# time-zone stuff -----------------------------------------------------------
	# determine the time-zone offset value (in the following order):
	# - if provided via the -tz parameter, use that value
	# - if a non-empty DEFAULT_TIME_ZONE_OFFSET was provided, use that value
	# - use the system's time-zone offset value
	if test "$TIME_ZONE_OFFSET" = ''
	then
		TIME_ZONE_OFFSET=`/bin/date +%z`
	else
		# do some basic validation of the time-zone offset value
		MSG="\nERROR: the time-zone offset '$TIME_ZONE_OFFSET' is not valid ... quitting${BEL}"

		if test \
			\( "`echo $TIME_ZONE_OFFSET | \
									/usr/bin/grep ^[+\-]*[0-9][0-9][0-9]$`" = '' \) -a \
			\( "`echo $TIME_ZONE_OFFSET | \
									/usr/bin/grep ^[+\-]*[0-9][0-9][0-9][0-9]$`" = '' \)
		then
			echo "$MSG"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		else
			if test \( $TIME_ZONE_OFFSET -gt 2400 \) -o \
					  \( $TIME_ZONE_OFFSET -lt -2400 \)
			then
				echo "$MSG"
				/bin/rm -f "$WORK_FILE_PATHNAME"
				exit 1
			fi
		fi
	fi

	# determine the hours and the minutes portions of the timezone offset value
	TZ_OFFSET_HRS=`
			echo "$TIME_ZONE_OFFSET" | /usr/bin/sed -e 's/^\(.*\)[0-9][0-9]$/\1/'`
	TZ_OFFSET_MINS=`
			echo "$TIME_ZONE_OFFSET" | /usr/bin/sed -e 's/^.*\([0-9][0-9]\)$/\1/'`
	# compute the time-zone offset in seconds
	TZ_OFFSET_SEC=`
			/bin/expr \( $TZ_OFFSET_HRS \* 3600 \) + \( $TZ_OFFSET_MINS \* 60 \)`
fi

# if just showing help, the formulas stuff can be skipped
if test "$SHOW_HELP" != '1'
then
	# formulas stuff ------------------------------------------------------------
	# get the formulas
	if test "$FORMULAS_PATHNAME" != '' # formulas are via the -fp script argument
	then
		if test ! -f "$FORMULAS_PATHNAME"
		then
			echo "\nERROR: the formulas file"
			echo "$FORMULAS_PATHNAME"
			echo "could not be found ... quitting${BEL}"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		fi

		FULL_FORMULAS_SPEC=`/bin/cat "$FORMULAS_PATHNAME"`
	elif test "$FORMULAS_FILE" != ''  # formulas via file set in script variable
	then
		if test ! -f "$FORMULAS_FILE"
		then
			echo "\nERROR: The file set in the script's FORMULAS_FILE variable"
			echo "$FORMULAS_FILE"
			echo "could not be found ... quitting${BEL}"
			/bin/rm -f "$WORK_FILE_PATHNAME"
			exit 1
		fi

		FULL_FORMULAS_SPEC=`/bin/cat "$FORMULAS_FILE"`
		FORMULAS_PATHNAME="$FORMULAS_FILE"
	elif test "$LOCAL_FORMULAS" != ''  # formulas are defined in script variable
	then
		FULL_FORMULAS_SPEC="$LOCAL_FORMULAS"
		FORMULAS_PATHNAME="$0"
	fi

	if test "$OPEN_CONFIG_FILES" != '1'
	then
		# from the full formulas specification, remove:
		# - comment and empty or whitespace-only lines
		# - single quotes and question marks
		# - whitespace at start/end of lines
		FULL_FORMULAS_SPEC=`echo "$FULL_FORMULAS_SPEC" | /usr/bin/tr "\t" ' ' | \
														/usr/bin/sed -e 's/^[ ]*#.*$//'     \
																		 -e 's/[\?'"'"']//g' |  \
															/usr/bin/sed -e '/^[ ]*$/d'      \
																			 -e 's/^ [ ]*//'     \
																			 -e 's/ [ ]*$//'`
		#echo "\nFULL_FORMULAS_SPEC\n$FULL_FORMULAS_SPEC\n"

		# for the full list of formulas:
		# - remove any whitespace adjacent to the tilde separators
		# - flatten the list to a single line by changing EOL/newlines to tildes
		# - remove the final tilde at the end of the flattened list/line
		FULL_FORMULAS_SPEC_AS_LINE=`echo "$FULL_FORMULAS_SPEC" | \
			/usr/bin/awk -F '~' '
				# defines leading/trailing whitespace-trimming functions
				function ltrim(s) { sub(/^[ \t]+/, "", s); return s }
				function rtrim(s) { sub(/[ \t]+$/, "", s); return s }
				function trim(s) { return rtrim(ltrim(s)); }

				{
					if ((length($1) != 0) && (length($2) != 0) && 
						 (length($3) != 0) && (substr($1, 1, 1) != "#")) {
						printf("%s~%s~%s?", trim($1), trim($2), trim($3))
					}
				}' | /usr/bin/sed -e 's/\?$//'`
		#echo "\nFULL_FORMULAS_SPEC_AS_LINE (unflattened)"
		#echo "$FULL_FORMULAS_SPEC_AS_LINE" | /usr/bin/tr '?' "\n"

		# create a tilde-separated list containing all available formula numbers
		ALL_FORMULA_NUMS=`echo "$FULL_FORMULAS_SPEC" | /usr/bin/awk -F '~' '
			BEGIN {
				isFirstItem = 0
				separator = ""
			}
			{
				if ((length($1) != 0) && (length($2) != 0) && (length($3) != 0) &&
					 (substr($1, 1, 1) != "#")) {
					printf("%s%s", separator, $1)
					if (isFirstItem == 0) { separator = "~"; isFirstItem = 1 }
				}
			}'`
		#echo "\nALL_FORMULA_NUMS";echo "'$ALL_FORMULA_NUMS'" | /usr/bin/tr '~' ','

		# if applicable, show the list of available formulas then, if required,
		# get the list of formula numbers to be applied to the CSV fields
		if test \( "$SHOW_FORMULAS_AND_QUIT" = '1' \) -o \
				  \( \( "$FORMULA_NUM_LIST" = '' \) -a \
					  \( "$SHOW_INFO_AND_QUIT" != '1' \) \)
		then
			# show the list of formulas
			echo "\nThe complete list of formulas available ..."
			echo 'Formula  Formula'
			echo 'Number    Name'
			echo '------  -------------------'
			echo "$FULL_FORMULAS_SPEC_AS_LINE" | /usr/bin/tr '?' "\n" | \
								/usr/bin/awk -F '~' '{ printf("%5d   %s\n", $1, $2) }'

			if test "$SHOW_FORMULAS_AND_QUIT" = '1'
			then
				/bin/rm -f "$WORK_FILE_PATHNAME"
				exit 0
			fi

			# get the formulas to be applied
			echo '-----'

			if test "$RESULT_CSV_FIELD_NAMES" != ''
			then
				echo "\nResult-file CSV field names:"
				echo "$RESULT_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
			fi

			if test "$SELECTED_SOURCE_CSV_FIELD_NUMS" != ''
			then
				echo "\nSelected CSV field numbers:"
				echo "$SELECTED_SOURCE_CSV_FIELD_NUMS" | /usr/bin/sed -e 's/,/ , /g'
			fi

			echo "\nEnter a comma-separated list of formula numbers to be applied to selected fields:"
			echo "(an empty entry or a '0'/zero entry indicates no formula for a field)"
			read FORMULA_NUM_LIST
		fi

		# if required, partially validate the formula-number list
		if test "$FORMULA_NUM_LIST" = '-1'  # no formula number list was supplied
		then
			FORMULA_NUM_LIST=''
		else
			# remove whitespace and end-of-line comma
			FORMULA_NUM_LIST=`echo "$FORMULA_NUM_LIST" | \
												/usr/bin/sed -e 's/[\ \	]//g' -e 's/,$//'`
			#echo "\nFORMULA_NUM_LIST = '$FORMULA_NUM_LIST'"

			# partially validate field-number list (ensure only has #'s & commas)
			MSG="\nERROR: the formula list\n$FORMULA_NUM_LIST\nis not valid -- quitting${BEL}"

			if test "`echo $FORMULA_NUM_LIST | \
														/usr/bin/sed -e 's/[0-9\,]//g'`" != ''
			then
				echo "$MSG"
				echo "\nThe available formula numbers are:"
				echo "$ALL_FORMULA_NUMS" | /usr/bin/tr '~' ','
				/bin/rm -f "$WORK_FILE_PATHNAME"
				exit 1
			fi
		fi
	fi
fi

# CSV_FIELDS stuff -------------------------------------------------------------
# get the location of the CSV fields specifications
if test "$FIELDS_SPEC_PATHNAME" != ''  # fields defs via -fsp script argument
then
	if test ! -f "$FIELDS_SPEC_PATHNAME"
	then
		echo "\nERROR: the CSV fields specification file"
		echo "$FIELDS_SPEC_PATHNAME"
		echo "could not be found ... quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi
elif test "$THE_CSV_FIELDS_SPEC_FILE" != ''  # fields defs filename is in script
then
	if test ! -f "$THE_CSV_FIELDS_SPEC_FILE"
	then
		echo "\nERROR: The file set in the script's THE_CSV_FIELDS_SPEC_FILE variable"
		echo "$THE_CSV_FIELDS_SPEC_FILE"
		echo "could not be found ... quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	FIELDS_SPEC_PATHNAME="$THE_CSV_FIELDS_SPEC_FILE"
else
	FIELDS_SPEC_PATHNAME="$0"
fi
#echo "\nFIELDS_SPEC_PATHNAME = '$FIELDS_SPEC_PATHNAME'"

# now know both the fields IDs and formulas files ... open them
if test "$OPEN_CONFIG_FILES" = '1'
then
	if test "$TEXT_EDITOR" = ''
	then
		/usr/bin/open -e "$FIELDS_SPEC_PATHNAME"
		/usr/bin/open -e "$FORMULAS_PATHNAME"
	else
		/usr/bin/open -a "$TEXT_EDITOR" "$FIELDS_SPEC_PATHNAME"
		/usr/bin/open -a "$TEXT_EDITOR" "$FORMULAS_PATHNAME"
	fi

	echo "\nFile opening(s) attempted ... quitting"
	exit 0
fi

# get fields suffix(s) of currently defined "CSV_FIELDS_<fields ID>" values
DEFINED_CSV_FIELDS_NAME_SUFFIXES=`
 /usr/bin/grep '^[\ \	]*CSV_FIELDS_[a-zA-Z0-9_][a-zA-Z0-9_]*\=' \
 																	"$FIELDS_SPEC_PATHNAME" | \
  /usr/bin/sed -e 's/^[\ \	]*CSV_FIELDS_\([a-zA-Z0-9_][a-zA-Z0-9_]*\)\=.*$/\1/' |\
   /usr/bin/tr "\n" ','`
#echo "\nDEFINED_CSV_FIELDS_NAME_SUFFIXES = '$DEFINED_CSV_FIELDS_NAME_SUFFIXES'"

# if required, finish showing the help info (and exit)
if test "$SHOW_HELP" = '1'
then
	CURRENT_SUFFIXES=`echo "$DEFINED_CSV_FIELDS_NAME_SUFFIXES" | \
												/usr/bin/sed -e 's/,$//' -e 's/,/ , /g'`

	if test "$CURRENT_SUFFIXES" = ''
	then
		echo "  ${bON}(none)${bOFF}"
	else
		echo "  ${bON}$CURRENT_SUFFIXES${bOFF}"
	fi

	exit
fi

FIELD_NAME_PARTS=''
NEW_FIELD_NAMES=''
FIELD_FORMAT_LIST=''

# if required, get the CSV fields specification and extract the specs
if test "$THE_CSV_FIELDS_SPEC_ID" != ''
then
	THE_CSV_FIELDS_SPEC=''

	# validate the <fields ID> value
	THE_CSV_FIELDS_SPEC_ID=`echo "$THE_CSV_FIELDS_SPEC_ID" | \
														/usr/bin/sed -e 's/[^a-zA-Z0-9_]//g'`
	#echo "THE_CSV_FIELDS_SPEC_ID = '$THE_CSV_FIELDS_SPEC_ID'"

	if test "`echo ,$DEFINED_CSV_FIELDS_NAME_SUFFIXES | \
											/usr/bin/grep ,$THE_CSV_FIELDS_SPEC_ID,`" = ''
	then
		echo "\nERROR: there is no 'CSV_FIELDS_$THE_CSV_FIELDS_SPEC_ID' specification"

		if test "$FIELDS_SPEC_PATHNAME" = "$0"  # using script-based definitions
		then
			echo "in the "`/usr/bin/basename "$0"`" script ... quitting${BEL}"
		else
			echo 'in the file'
			echo "$FIELDS_SPEC_PATHNAME"
			echo "... quitting${BEL}"
		fi

		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# get the named/suffix'd CSV_FIELDS specification
	if test "$FIELDS_SPEC_PATHNAME" = "$0"  # using script-based definitions
	then
		THE_CSV_FIELDS_SPEC=`eval echo \"\\$CSV_FIELDS_\$THE_CSV_FIELDS_SPEC_ID\"`
	else
		THE_CSV_FIELDS_SPEC_NAME="CSV_FIELDS_$THE_CSV_FIELDS_SPEC_ID"

		# extract the lines that make up the CSV fields specification
		THE_CSV_FIELDS_SPEC=`/bin/cat "$FIELDS_SPEC_PATHNAME" | /usr/bin/awk '
			BEGIN {
				theCSVfieldsSpecName = "'"$THE_CSV_FIELDS_SPEC_NAME"'"
				state = 0  # 0 = finding start, 1 = at start, 2 = finding end
			}
			{
				if (match($0, "^[ \t]*"(theCSVfieldsSpecName)"=") != 0) {state = 1}

				if (state > 0) {
					print $0

					if ((state > 1) && (match($0, "'\''")) != 0) { exit }
					state = 2
				}
			}
		' | \
		# remove CSV_FIELD_<fields ID> variable, equals sign & single-quotes
		/usr/bin/sed -e "s/^[ \	]*$THE_CSV_FIELDS_SPEC_NAME\=.*\'//" -e "s/'//"`
	fi
	#echo "\nTHE_CSV_FIELDS_SPEC_ID = '$THE_CSV_FIELDS_SPEC_ID'"
	#echo "\nTHE_CSV_FIELDS_SPEC_NAME = '$THE_CSV_FIELDS_SPEC_NAME'"
	#echo "\nTHE_CSV_FIELDS_SPEC:\n$THE_CSV_FIELDS_SPEC\n"

	# for the CSV fields specification:
	# - change all tabs to a single space
	# - remove comment lines
	# - remove the whitespace at the start/end of entry-lines
	# - remove the whitespace adjacent to the comma field-separators
	# - change any tilde characters to dash characters
	# - change any bang/question-mark characters to dash characters
	# - remove the end-of-entry comment fields
	# - remove any entry-lines that do not have a <field name part> value
	# - remove any empty or whitespace-only entry-lines
	# - escape any double-quote characters
	# - remove any single-quote characters
	# - change all end-of-line/newline characters to tilde characters (flattens)
	# - remove the end-of-line tilde character from the flattened specification
	THE_CSV_FIELDS_SPEC=`echo "$THE_CSV_FIELDS_SPEC" | /usr/bin/tr "\t" ' ' | \
															/usr/bin/sed -e '/^[ ]*#/d'     \
																			 -e '/^[ ]*#/d'     \
																			 -e 's/^[ ]*//'     \
																			 -e 's/[ ]*$//'     \
																			 -e 's/[ ]*,/,/g'   \
																			 -e 's/,[ ]*/,/g'   \
																			 -e 's/~/-/g'       \
																			 -e 's/\?/-/g'      \
																			 -e 's/[, ]*#.*$//' \
																			 -e '/^,/d'         \
																			 -e '/^[ ]*$/d'     \
																			 -e 's/\"/\\\"/g' | \
																/usr/bin/tr -d "'" |         \
																	/usr/bin/tr "\n" '~' |    \
																		/usr/bin/sed -e 's/~$//'`
	#echo "\nTHE_CSV_FIELDS_SPEC (unflattened):"
	#echo "\n$THE_CSV_FIELDS_SPEC\n" | /usr/bin/tr '~' "\n"

	# create a comma-separated list of the selected original/source CSV field
	# numbers -- the source CSV field numbers are found by looking up the number
	# of each source CSV field that corresponds each <field name part> in the
	# CSV_FIELDS specification
	SELECTED_SOURCE_CSV_FIELD_NUMS=`\
		/usr/bin/awk \
		 -v sourceCSVfieldsNumNameList="$SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE" '
			BEGIN {
				# SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain double-quotes
				# NOTE: since SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain
				#       double-quotes, it must be loaded via the -v argument (unless
				#       quotes are escaped)

				# create an array of the source CSV field num/name entries and get
				# the number of source CSV fields
				numSourceCSVfields = split(sourceCSVfieldsNumNameList, \
													sourceCSVfieldNumNameEntry, "~")

				# create an associative array to enable a source CSV field number to
				# be looked up (i.e., indexed) by a CSV field name so:
				# CSVfieldNum[<CSV field name>] is the <CSV field number>
				for (i = 1; i <= numSourceCSVfields; i++) {
					split(sourceCSVfieldNumNameEntry[i], CSVfieldNumName, ",")
					CSVfieldNum[CSVfieldNumName[2]] = CSVfieldNumName[1] #name>field#
				}

				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the number/position of each CSV
				# field that corresponds to the CSV field name in the CSV_FIELDS
				# specification
				isFirstField = 1
				thisCSVfieldNum = -1

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the field-name part from the CSV_FIELDS specification
					fieldNamePart = FIELDSspecInfo[1]

					# find the corresponding full/original CSV field name
					prevIndex = 999
					prevPortionMatched = 0 ;  # prev portion of source CSV name matched
					theFoundCSVfieldName = ""

					# find "best match" (see below for meaning of "best match")
					for (j = 1; j <= numSourceCSVfields; j++) {
						portionOfCSVname = 0.0;
						split(sourceCSVfieldNumNameEntry[j], CSVfieldNumName, ",")
						fullCSVfieldName = CSVfieldNumName[2]

						# check for exact match
						if (fullCSVfieldName == fieldNamePart) {
							theFoundCSVfieldName = fullCSVfieldName
							break
						}

						# search for a match of the <field name part> in the source
						# CSV field name
						currIndex = index(fullCSVfieldName, fieldNamePart)

						if (currIndex != 0) {
							# determine the portion the source CSV field name that the
							# <field name part> constitutes
							portionOfCSVname = \
										length(fieldNamePart) / length(fullCSVfieldName)

							# if applicable, update the "found field name" if this is a
							# "better" match -- i.e.:
							# - the current match is more left-most than the previous
							#   match (or non-match)
							#   OR :
							# - the current match is as left-most as the previous match
							#   (or non-match) AND
							# - the current <field name part> is a greater portion of
							#   the source CSV field name that was the previous match
							#   (or non-match)
							if ((currIndex < prevIndex) ||
								 ((currIndex == prevIndex) &&
								  (portionOfCSVname > prevPortionMatched))) {
								prevIndex = currIndex
								prevPortionMatched = portionOfCSVname
								theFoundCSVfieldName = fullCSVfieldName
							}
						}
					}

					# get the number/position of the CSV field that corresponds to
					# the name of the CSV field name from the CSV_FIELDS spec
					thisCSVfieldNum = CSVfieldNum[theFoundCSVfieldName]
					# for debug ... fails if names have quote-requiring characters
					#system("echo "fieldNamePart" x "theFoundCSVfieldName" >&2")

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					# report invalid field numbers
					if (length(thisCSVfieldNum) == 0) {
						system("echo >&2 ; echo ERROR: the CSV file has no field matching the field name part \\\\\""fieldNamePart"\\\\\" >&2")
						thisCSVfieldNum = -1
					}

					printf("%d", thisCSVfieldNum)
				}
			}'`
	#echo "\nSELECTED_SOURCE_CSV_FIELD_NUMS = '$SELECTED_SOURCE_CSV_FIELD_NUMS'"

	if test "`echo $SELECTED_SOURCE_CSV_FIELD_NUMS | /usr/bin/fgrep -e '-1'`" != ''
	then
		echo "\nError(s) encountered ... quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# create a comma-separated list of the selected original/source CSV field
	# names by looking up the name of each source CSV field that corresponds to
	# each <field name part> in the CSV_FIELDS specification
	SELECTED_SOURCE_CSV_FIELD_NAMES=`\
		/usr/bin/awk \
		 -v sourceCSVfieldsNumNameList="$SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE" '
			BEGIN {
				# SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain double-quotes
				# NOTE: since SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain
				#       double-quotes, it must be loaded via the -v argument (unless
				#       quotes are escaped)

				# create an array of the source CSV field num/name entries and get
				# the number of source CSV fields
				numSourceCSVfields = split(sourceCSVfieldsNumNameList, \
													sourceCSVfieldNumNameEntry, "~")

				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the name of each CSV field that
				# corresponds to the CSV field name in the CSV_FIELDS specification
				isFirstField = 1
				origCSVfieldName = ""

				# get the name of the source CSV field that corresponds to the
				# <field name part> from the CSV_FIELDS specification
				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get <field name part> from the CSV_FIELDS specification
					fieldNamePart = FIELDSspecInfo[1]

					# find the corresponding full/original CSV field name
					prevIndex = 999
					prevPortionMatched = 0 ;  # prev portion of source CSV name matched
					theFoundCSVfieldName = ""

					# find "best match" (see below for meaning of "best match")
					for (j = 1; j <= numSourceCSVfields; j++) {
						portionOfCSVname = 0.0;
						split(sourceCSVfieldNumNameEntry[j], CSVfieldNumName, ",")
						fullCSVfieldName = CSVfieldNumName[2]

						# check for exact match
						if (fullCSVfieldName == fieldNamePart) {
							theFoundCSVfieldName = fullCSVfieldName
							break
						}

						# search for a match of the <field name part> in the source
						# CSV field name
						currIndex = index(fullCSVfieldName, fieldNamePart)

						if (currIndex != 0) {
							# determine the portion the source CSV field name that the
							# <field name part> constitutes
							portionOfCSVname = \
										length(fieldNamePart) / length(fullCSVfieldName)

							# for debugging
							#fieldPart = fieldNamePart;
							#CSVfieldName = fullCSVfieldName;
							# quote any applicable characters in the names ...
							# e.g., /"|\|/ for double quotes and vertical bars
							#gsub(/"/, " ", fieldPart);
							#gsub(/"|\|/, " ", CSVfieldName);
							#system("echo >&2 ; echo fieldNamePart = "fieldPart", fullCSVfieldName = "CSVfieldName" >&2")
							#system("echo currIndex = "currIndex", prevIndex = "prevIndex" >&2")
							#system("echo portionOfCSVname = "portionOfCSVname", prevPortionMatched = "prevPortionMatched" >&2")

							# if applicable, update the "found field name" if this is a
							# "better" match -- i.e.:
							# - the current match is more left-most than the previous
							#   match (or non-match)
							#   OR :
							# - the current match is as left-most as the previous match
							#   (or non-match) AND
							# - the current <field name part> is a greater portion of
							#   the source CSV field name that was the previous match
							#   (or non-match)
							if ((currIndex < prevIndex) ||
								 ((currIndex == prevIndex) &&
								  (portionOfCSVname > prevPortionMatched))) {
								prevIndex = currIndex
								prevPortionMatched = portionOfCSVname
								theFoundCSVfieldName = fullCSVfieldName
							}

							# for debugging
							#foundFieldName = theFoundCSVfieldName;
							# quote any applicable characters in the names ...
							# e.g., /"|\|/ for double quotes and vertical bars
							#gsub(/"|\|/, " ", foundFieldName);
							#system("echo theFoundCSVfieldName = "foundFieldName" >&2")
						}
					}

					origCSVfieldName = theFoundCSVfieldName

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					# report invalid <field name part> entries
					if (length(origCSVfieldName) == 0) {
						system("echo >&2 ; echo ERROR: the CSV file has no field matching the field name part \\\\\""fieldNamePart"\\\\\" >&2")
						origCSVfieldName = "--err--"
					}

					printf("%s", origCSVfieldName)
				}
			}'`
	#echo "\nSELECTED_SOURCE_CSV_FIELD_NAMES = '$SELECTED_SOURCE_CSV_FIELD_NAMES'"

	if test "`echo $SELECTED_SOURCE_CSV_FIELD_NAMES | \
															/usr/bin/fgrep -e '--err--'`" != ''
	then
		echo "\nError(s) encountered ... quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# create a comma-separated list of the <field name part> entries in the
	# CSV_FIELDS specification
	FIELD_NAME_PARTS=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the <field name part> entries
				# from the CSV_FIELDS specification
				isFirstField = 1
				fieldNamePart = ""

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the field name part from the CSV_FIELDS specification
					fieldNamePart = FIELDSspecInfo[1]

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%s", fieldNamePart)
				}
			}'`
	#echo "\nFIELD_NAME_PARTS = '$FIELD_NAME_PARTS'"

	# create a comma-separated list of the <new field name> entries in the
	# CSV_FIELDS specification
	NEW_FIELD_NAMES=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the <new field name> entries
				# from the CSV_FIELDS specification
				isFirstField = 1
				newFieldName = ""

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the new field name from the CSV_FIELDS specification
					newFieldName = FIELDSspecInfo[2]

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%s", newFieldName)
				}
			}'`
	#echo "\nNEW_FIELD_NAMES = '$NEW_FIELD_NAMES'"

	# create a comma-separated list of the CSV field names for the result file
	RESULT_CSV_FIELD_NAMES=`\
		/usr/bin/awk \
		 -v sourceCSVfieldsNumNameList="$SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE" '
			BEGIN {
				# SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain double-quotes
				# NOTE: since SOURCE_CSV_FIELD_NUM_NAME_LIST_AS_LINE may contain
				#       double-quotes, it must be loaded via the -v argument (unless
				#       quotes are escaped)

				# create an array of the source CSV field num/name entries and get
				# the number of source CSV fields
				numSourceCSVfields = split(sourceCSVfieldsNumNameList, \
													sourceCSVfieldNumNameEntry, "~")

				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the CSV field names (header) from
				# the CSV_FIELDS specification, to be used in the results file
				isFirstField = 1
				thisNewCSVFieldName = ""

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the <new field name> -- if empty, use the <field name part>
					if (length(FIELDSspecInfo[2]) != 0) {
						thisNewCSVFieldName = FIELDSspecInfo[2]
					}
					else { thisNewCSVFieldName = FIELDSspecInfo[1] }

					# if the new field name is a bang/exclamation (!) then look up
					# and use the source CSV field name
					if (thisNewCSVFieldName == "!") {
						# get <field name part> from the CSV_FIELDS specification
						fieldNamePart = FIELDSspecInfo[1]

						for (j = 1; j <= numFIELDSspecEntries; j++) {
							# find the corresponding full/original CSV field name
							prevIndex = 999
							prevPortionMatched = 0 ;  # prev portion of source CSV name matched
							theFoundCSVfieldName = ""

							# find "best match" (see below for meaning of "best match")
							for (k = 1; k <= numSourceCSVfields; k++) {
								portionOfCSVname = 0.0;
								split(sourceCSVfieldNumNameEntry[k], \
										CSVfieldNumName, ",")
								fullCSVfieldName = CSVfieldNumName[2]

								# check for exact match
								if (fullCSVfieldName == fieldNamePart) {
									theFoundCSVfieldName = fullCSVfieldName
									break
								}

								# search for a match of the <field name part> in the
								# source CSV field name
								currIndex = index(fullCSVfieldName, fieldNamePart)

								if (currIndex != 0) {
									# determine the portion the source CSV field name
									# that the <field name part> constitutes
									portionOfCSVname = \
										length(fieldNamePart) / length(fullCSVfieldName)

									# if applicable, update the "found field name" if
									# this is a "better" match -- i.e.:
									# - the current match is more left-most than the
									#   previous match (or non-match)
									#   OR :
									# - the current match is as left-most as the previous
									#   match (or non-match) AND
									# - the current <field name part> is a greater
									#   portion of the source CSV field name that was the
									#   previous match (or non-match)
									if ((currIndex < prevIndex) ||
										 ((currIndex == prevIndex) &&
										  (portionOfCSVname > prevPortionMatched))) {
										prevIndex = currIndex
										prevPortionMatched = portionOfCSVname
										theFoundCSVfieldName = fullCSVfieldName
									}
								}
							}

							thisNewCSVFieldName = theFoundCSVfieldName
						}

						# if no match was found, use the <field name part> value
						if (theFoundCSVfieldName == "!") {
							thisNewCSVFieldName = fieldNamePart
						}
					}

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%s", thisNewCSVFieldName)
				}
			}'`
	#echo "\nRESULT_CSV_FIELD_NAMES = '$RESULT_CSV_FIELD_NAMES'"

	# create a comma-separated list of the CSV field formats for the result file
	FIELD_FORMAT_LIST=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the field formats from the
				# CSV_FIELDS specification
				isFirstField = 1
				thisCSVfieldFormat = ""

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the field <format spec> -- if empty, set to %s
					if (length(FIELDSspecInfo[3]) != 0) {
						thisCSVfieldFormat = FIELDSspecInfo[3]
					}
					else { thisCSVfieldFormat = "%s" }

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%s", thisCSVfieldFormat)
				}
			}'`
	#echo "\nFIELD_FORMAT_LIST = '$FIELD_FORMAT_LIST'"

	# create a comma-separated list of the required-field flags
	REQUIRED_FIELDS_FLAGS=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the required-fields flags from
				# the CSV_FIELDS specification
				isFirstField = 1
				thisRequiredFieldFlag = 0

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the required-field flag -- if empty or not 1, set it to 0
					if (length(FIELDSspecInfo[4]) != 0) {
						thisRequiredFieldFlag = FIELDSspecInfo[4]
						if (thisRequiredFieldFlag != 1) { thisRequiredFieldFlag = 0 }
					}
					else { thisRequiredFieldFlag = 0 }

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%d", thisRequiredFieldFlag)
				}
			}'`
	#echo "\nREQUIRED_FIELDS_FLAGS = '$REQUIRED_FIELDS_FLAGS'"

	# create a comma-separated list of the sort-order numbers
	SORT_ORDER_LIST=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the sort-order numbers from the
				# CSV_FIELDS specification
				isFirstField = 1
				thisSortOrderNum = 0

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the sort-order number -- if empty, set it to 0
					if (length(FIELDSspecInfo[5]) != 0) {
						thisSortOrderNum = FIELDSspecInfo[5]
					}
					else { thisSortOrderNum = 0 }

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%d", thisSortOrderNum)
				}
			}'`
	#echo "\nSORT_ORDER_LIST = '$SORT_ORDER_LIST'"

	# create a comma-separated list of the applied formula numbers
	FORMULA_NUM_LIST=`\
		/usr/bin/awk '
			BEGIN {
				# load the selected/named CSV_FIELDS specification into awk
				theFIELDSspec = "'"$THE_CSV_FIELDS_SPEC"'"

				# create an array of fields-spec entries & get the number of entries
				numFIELDSspecEntries = split(theFIELDSspec, FIELDSspecEntry, "~")

				# output a comma-separated list of the formula numbers from the
				# CSV_FIELDS specification
				isFirstField = 1
				thisFormulaNum = 0

				for (i = 1; i <= numFIELDSspecEntries; i++) {
					split(FIELDSspecEntry[i], FIELDSspecInfo, ",")

					# get the formula number -- if empty, set to 0
					if (length(FIELDSspecInfo[6]) != 0) {
						thisFormulaNum = FIELDSspecInfo[6]
					}
					else { thisFormulaNum = 0 }

					if (isFirstField == 1) { isFirstField = 0 }
					else { printf(",") }

					printf("%d", thisFormulaNum)
				}
			}'`
	#echo "\nFORMULA_NUM_LIST = '$FORMULA_NUM_LIST'"
fi

# determine whether any non-"%s" field-formats are to be applied
if test "`echo $FIELD_FORMAT_LIST | \
											/usr/bin/sed -e 's/%s//g' -e 's/,//g'`" != ''
then
	FIELD_FORMATS_ARE_USED='1'
fi

# determine whether any (non-zero) sort-order numbers are provided
if test "`echo $SORT_ORDER_LIST | /usr/bin/sed -e 's/[0\,]//g'`" != ''
then
	SORTING_IS_REQUIRED='1'
fi

# determine whether any (non-zero) formulas are to be applied
if test "`echo $FORMULA_NUM_LIST | /usr/bin/sed -e 's/[0\,]//g'`" != ''
then
	FORMULAS_ARE_USED='1'
fi

if test "`echo $REQUIRED_FIELDS_FLAGS | /usr/bin/sed -e 's/[0,]//g'`" = ''
then
	# there are no non-zero/non-empty required-fields flags
	REQUIRED_FIELDS_FLAGS=''
fi

# selected CSV fields stuff ----------------------------------------------------
# save current IFS (inter-field separator)
IFS_SAVE=$IFS

# set IFS to be a comma, only
IFS=','

# validate the selected CSV fields and create the fields portion of an awk print
# command to print the selected fields
NUM_RESULT_CSV_FIELDS=0
IS_FIRST_FIELD='1'
HEADER_PRINT_CMD=''
DATA_PRINT_CMD=''
HAD_INVALID_FIELD='0'
OPS_PREFIX=''
OPS_SUFFIX=''
CURR_JOIN_FIELD=0
PREV_FIELD_NUM=0

# generate a prefix and suffix for operations that may be required for each
# selected CSV field
if test \( "$FORMULAS_ARE_USED" = '1' \) -o \( "$REMOVE_QUOTES" = '1' \)
then
	# need to remove all double-quote characters
	OPS_PREFIX='deQuote('
	OPS_SUFFIX=')'
fi
if test "$DO_TRIM" = '1'
then
	OPS_PREFIX="$OPS_PREFIX"'trim('
	OPS_SUFFIX="$OPS_SUFFIX"')'
fi

if test "$JOIN_FIELDS" = '1'
then
	# get the next "to be joined" field in the field list
	CURR_JOIN_FIELD=`echo "$TO_BE_JOINED_FIELD_NUMS" | \
											/usr/bin/sed -e 's/^\([0-9][0-9]*\),.*$/\1/'`

	# remove the current entry fromt the "to be joined" field list
	TO_BE_JOINED_FIELD_NUMS=`echo "$TO_BE_JOINED_FIELD_NUMS" | \
											/usr/bin/sed -e 's/^[0-9][0-9]*,*\(.*\)$/\1/'`
fi

# generate the fields part of the awk command that will select the CSV fields
for FIELD_NUMBER in $SELECTED_SOURCE_CSV_FIELD_NUMS
do
	SEP=','
	NUM_RESULT_CSV_FIELDS=`/bin/expr $NUM_RESULT_CSV_FIELDS + 1`

	# ensure it's a valid field number
	if test \
		"`echo $ALL_ORIG_CSV_FIELD_NUMS | \
			/usr/bin/sed -e 's/^.*\('$FIELD_NUMBER'\).*$/\1/'`" != "$FIELD_NUMBER"
	then
		if test "$HAD_INVALID_FIELD" != '1'
		then
			echo
		fi

		echo "ERROR: the CSV field number $FIELD_NUMBER is not valid"
		HAD_INVALID_FIELD='1'
		continue
	fi
	#echo "\nCURR_JOIN_FIELD = $CURR_JOIN_FIELD  ~  FIELD_NUMBER = $FIELD_NUMBER"

	# if joining fields and this is a joined field, make field separator a space
	if test \( "$JOIN_FIELDS" = '1' \) -a \
			  \( "$CURR_JOIN_FIELD" = "$PREV_FIELD_NUM" \)
	then
		SEP='~'  # later changed to bar (|) for header and comma for data records

		# get the next "to be joined" field in the field list
		CURR_JOIN_FIELD=`echo "$TO_BE_JOINED_FIELD_NUMS" | \
											/usr/bin/sed -e 's/^\([0-9][0-9]*\),.*$/\1/'`

		# remove the current entry fromt the "to be joined" field list
		TO_BE_JOINED_FIELD_NUMS=`echo "$TO_BE_JOINED_FIELD_NUMS" | \
											/usr/bin/sed -e 's/^[0-9][0-9]*,*\(.*\)$/\1/'`
	fi

	if test "$IS_FIRST_FIELD" = '1'
	then
		IS_FIRST_FIELD='0'
		DATA_PRINT_CMD="${DATA_PRINT_CMD}$OPS_PREFIX"'$'"${FIELD_NUMBER}$OPS_SUFFIX"
	else
		# add comma field-separator
		DATA_PRINT_CMD="${DATA_PRINT_CMD}\"$SEP\"$OPS_PREFIX"'$'"${FIELD_NUMBER}$OPS_SUFFIX"
	fi

	PREV_FIELD_NUM="$FIELD_NUMBER"
done
#echo "\nNUM_RESULT_CSV_FIELDS = $NUM_RESULT_CSV_FIELDS"

# if joining fields, create the header print cmd and adjust the data print cmd
if test "$JOIN_FIELDS" = '1'
then
	HEADER_PRINT_CMD=`echo "$DATA_PRINT_CMD" | /usr/bin/sed -e 's/~/|/g'`
	DATA_PRINT_CMD=`echo "$DATA_PRINT_CMD" | /usr/bin/sed -e 's/~/ /g'`
fi
#echo "\nHEADER_PRINT_CMD = $HEADER_PRINT_CMD"
#echo "\nDATA_PRINT_CMD = $DATA_PRINT_CMD"

# field-sort/command stuff -----------------------------------------------------
FIELDS_TO_SORT=''

# if required, validate the specified sort fields and create the fields portion
# of a sort command to sort the records by the specified sort fields
if test "$SORTING_IS_REQUIRED" = '1'
then
	# exit if invalid CSV or sort fields were supplied
	if test "$HAD_INVALID_FIELD" = '1'
	then
		echo "\nThe available fields are:\n$ALL_ORIG_CSV_FIELD_NUMS"
		echo "\nError(s) encountered ... quitting${BEL}"
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 1
	fi

	# create the sort-key arguments for the sort command
	SORTED_FIELD_NUM_AND_SORT_ORDER_LIST=`
	echo "$SORT_ORDER_LIST" | \
	# create <field number, sort order> pairs
	/usr/bin/awk -F ',' '
	{
		for (i = 1; i <= NF; i++) {
			if ((length($i) != 0) && ($i != 0)) { printf( "%s,%s\n", i, $i) }
		}
	}' | \
	# sort the <field number, sort order> pairs by the sort-order values
	/usr/bin/sort -n -t ',' -k 2,2`
	
	FIELDS_TO_SORT=`
		echo "$SORTED_FIELD_NUM_AND_SORT_ORDER_LIST" | \
		# output the sort-key arguments for the sort command (1 arg per field)
		/usr/bin/awk -F ',' '{ printf("%s,", $1) }' | /usr/bin/sed -e 's/,$//'`
	#echo "\nFIELDS_TO_SORT = $FIELDS_TO_SORT"
	
	SORT_CMD_SORT_KEYS=`
		echo "$SORTED_FIELD_NUM_AND_SORT_ORDER_LIST" | \
		# output the sort-key arguments for the sort command (1 arg per field)
		/usr/bin/awk -F ',' '{ printf(" -k %s,%s", $1, $1) }'`
	#echo "\nSORT_CMD_SORT_KEYS = $SORT_CMD_SORT_KEYS"

	sCMD='/usr/bin/sort'"${REMOVE_DUPLICATES}${SORT_CASE_OPTION}${NUMERIC_COLLATION}${DO_MONTH_SORT}${SORT_DIRECTION}"' -t ","'"$SORT_CMD_SORT_KEYS"
else
	sCMD='/bin/cat'  # a no-sort pass-through
fi
#echo "\nsCMD = $sCMD"

# show selected fields hack ----------------------------------------------------
# this is somewhat of a hack ... but is _real_ useful when debugging  #;-))
if test "$SHOW_SELECTED_FIELDS" = '1'
then
	sCMD="$sCMD"' >> "$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"'
fi
#echo "\nsCMD = $sCMD"

# show info stuff --------------------------------------------------------------
# if required, output info about the various selections
if test "$SHOW_INFO" = '1'
then
	echo "\nResults file name: ${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"

	echo "\nThe complete list of fields in the source CSV file ..."
	echo ' Field  Field'
	echo 'Number  Name'
	echo '------  -------------------'
	echo "$SOURCE_CSV_FIELD_NUM_NAME_LIST" | \
								/usr/bin/awk -F ',' '{ printf("%5d   %s\n", $1, $2) }'

	if test "$THE_CSV_FIELDS_SPEC_ID" != ''
	then
		echo "\nThe location of the CSV_FIELDS specification is:"
		echo "$FIELDS_SPEC_PATHNAME"

		echo "\nThe CSV_FIELDS ID value: $THE_CSV_FIELDS_SPEC_ID"
	fi

	if test "$SELECTED_SOURCE_CSV_FIELD_NAMES" != ''
	then
		echo "\nThe list of selected source/original CSV field names:"
		echo "$SELECTED_SOURCE_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
	fi

	if test "$SELECTED_SOURCE_CSV_FIELD_NUMS" != ''
	then
		echo "\nThe list of selected source/original CSV field numbers:"
		echo "$SELECTED_SOURCE_CSV_FIELD_NUMS" | /usr/bin/sed -e 's/,/ , /g'
	fi

	if test "$JOIN_FIELDS" = '1'
	then
		echo "\nThe list of 'to be joined' CSV field numbers:"
		echo "$FULL_TO_BE_JOINED_FIELD_NUMS" | /usr/bin/sed -e 's/,/ , /g'
	fi

	if test "`echo $REQUIRED_FIELDS_FLAGS | /usr/bin/sed -e 's/[,0]//g'`" != ''
	then
		echo "\nThe list of required-field flags:"
		echo "$REQUIRED_FIELDS_FLAGS" | /usr/bin/sed -e 's/,/ , /g'
	fi

	if test "$SORTING_IS_REQUIRED" = '1'
	then
		echo "\nThe list of sort-order numbers:"
		echo "$SORT_ORDER_LIST" | /usr/bin/sed -e 's/,/ , /g'
		echo '... so records will be sorted by the following result field(s):' 
		echo "$FIELDS_TO_SORT" | /usr/bin/sed -e 's/,/ then /g'
	fi

	if test "$FORMULAS_ARE_USED" = '1'
	then
		echo "\nThe complete list of available formulas ..."
		echo 'Formula  Formula'
		echo 'Number    Name'
		echo '------  -------------------'
		echo "$FULL_FORMULAS_SPEC_AS_LINE" | /usr/bin/tr '?' "\n" | \
								/usr/bin/awk -F '~' '{ printf("%5d   %s\n", $1, $2) }'

		echo "\nThe location of the formulas is:"
		echo "$FORMULAS_PATHNAME"

		echo "\nThe list of applied formula numbers:"
		echo "$FORMULA_NUM_LIST" | /usr/bin/sed -e 's/,/ , /g'
	fi

	if test "$THE_CSV_FIELDS_SPEC_ID" != ''
	then
		if test "$FIELD_FORMATS_ARE_USED" = '1'
		then
			echo "\nThe list of result-file field-formats:"
			echo "$FIELD_FORMAT_LIST" | /usr/bin/sed -e 's/,/ , /g'
		fi

		if test "$RESULT_CSV_FIELD_NAMES" != ''
		then
			echo "\nThe list of result-file field names:"
			echo "$RESULT_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
		fi

		if test "`echo $NEW_FIELD_NAMES | /usr/bin/sed -e 's/,//g'`" != ''
		then
			echo "\nThe list of new field names in the CSV field spec:"
			echo "$NEW_FIELD_NAMES" | /usr/bin/sed -e 's/,/ , /g'
		fi

		if test "$FIELD_NAME_PARTS" != ''
		then
			echo "\nThe list of 'selector' field name parts in the CSV field spec:"
			echo "$FIELD_NAME_PARTS" | /usr/bin/sed -e 's/,/ , /g'
		fi
	fi

	if test "$FORMULAS_ARE_USED" = '1'
	then
		echo "\nThe timezone offset is $TZ_OFFSET_HRS:$TZ_OFFSET_MINS ($TZ_OFFSET_SEC seconds)"
	fi

	if test "$SHOW_INFO_AND_QUIT" = 1
	then
		/bin/rm -f "$WORK_FILE_PATHNAME"
		exit 0
	fi

	echo
fi

# field extraction stuff -------------------------------------------------------
echo "\n${bON}Extracting selected fields ...${bOFF}"

# show any notable conditions
if test "$CHANGE_SEPARATOR" != ''
then
	if test "$CHANGE_SEPARATOR" = '	'
	then
		SEP='tab'
	elif test "$CHANGE_SEPARATOR" = ' '
	then
		SEP='space'
	else
		SEP="'$CHANGE_SEPARATOR'"
	fi

	echo "\nNOTE: $SEP characters are being replaced with comma field-separators"
fi

if test "$REMOVE_QUOTES" = '1'
then
	echo "\nNOTE: double-quote removal is being performed"
fi

if test "$FILL_EMPTY_FIELDS" = '1'
then
	echo "\nNOTE: copy-forward field-filling is being performed"
fi

if test \( "$DO_TRIM" != '1' \) -a \( "$JOIN_FIELDS" != '1' \)
then
	echo "\nNOTE: whitespace before/after source field values is NOT being removed"
fi

if test \( "$SORT_CASE_OPTION" = '' \) -a\
		  \( "$NUMERIC_COLLATION" != '' \)
then
	# doing case-sensitive sorting but using numeric collating rules
	echo "\nNOTE: case-sensitive sorting (-cs) may also require alphabetic collating rules (-sa)"
fi

# warn about certain options that are being overridden or ignored
if test \( "$RESULTS_AS_FORMULAS" = '1' \) -a \( "$FORMULAS_ARE_USED" != '1' \)
then
	echo "\nNOTE: because no formulas are being used, the -sf option is ignored"
fi

if test \( "$SHOW_SELECTED_FIELDS" = '1' \) -a \( "$RESULTS_AS_FORMULAS" = '1' \)
then
	echo "\nNOTE: because selected fields are to be shown (-ss), the -sf option is ignored"
fi

if test "$THE_CSV_FIELDS_SPEC_ID" != ''
then
	# if required, warn about options/arguments that are overridden when using
	# a CSV_FIELDS specification (i.e., -fl, -fr, -so and -fnl)
	OVERRIDDEN_ARGS=''

	if test "$HAVE_fl_ARG" = '1'
	then
		OVERRIDDEN_ARGS="$OVERRIDDEN_ARGS"', -fl'
	fi

	if test "$HAVE_fr_ARG" = '1'
	then
		OVERRIDDEN_ARGS="$OVERRIDDEN_ARGS"', -fr'
	fi

	if test "$HAVE_so_ARG" = '1'
	then
		OVERRIDDEN_ARGS="$OVERRIDDEN_ARGS"', -so'
	fi

	if test "$HAVE_fnl_ARG" = '1'
	then
		OVERRIDDEN_ARGS="$OVERRIDDEN_ARGS"', -fnl'
	fi

	if test "$OVERRIDDEN_ARGS" != ''
	then
		OVERRIDDEN_ARGS=`echo "$OVERRIDDEN_ARGS" | /usr/bin/sed -e 's/^, //'`
		echo "\nNOTE: because fields ID '$THE_CSV_FIELDS_SPEC_ID' was provided, these argument(s) are ignored: $OVERRIDDEN_ARGS"
	fi
fi

/bin/echo -n > "$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"

# define the command collection that will be applied after the fields are
# extracted and the major processing is done
if test \( "$SHOW_SELECTED_FIELDS" = '1' \) -o \( "$JOIN_FIELDS" = '1' \)
then
	# set a no-change pass-through
	fCMD='/bin/cat'
elif test \( "$FORMULAS_ARE_USED" = '1' \) -a \( "$RESULTS_AS_FORMULAS" = '1' \)
then
	# allow the intermediate results to be output (useful when testing formulas)
	# test parts of the output by piping it through the following commands:
	# echo '<copy/pasted output stuff>' | \
	#            					   bc -l | tr -d "\n" | tr "~" "," | tr "?" "\n"
	fCMD='/bin/cat'  # no-change pass-through
elif test \( "$FORMULAS_ARE_USED" = '1' \) -a \
			 \( "$FIELD_FORMATS_ARE_USED" = '1' \)
	then
		# both formulas and field-formats are being used so process the formulas
		# via bc, do some cleanup processing, then perform the field formatting
		fCMD='# process the output using the bc utility to "do the math"
				/usr/bin/bc -l | \
				# get rid of the newlines added by bc
				/usr/bin/tr -d "\n" | \
				# translate the bcEOF-generated tilde field separators into commas
				/usr/bin/tr "~" "," | \
				# translate the added ? EOL characters into newlines
				/usr/bin/tr "?" "\n" | \
				# get rid of the backslash characters added by bc
				/usr/bin/tr -d "\\" | \
				# remove empty line at the end of the file
				/usr/bin/sed -e "/^[ \	]*$/d" | \
				# format the results fields
				/usr/bin/awk -F ',' '"'"'
					# define a function that tries to determine the output length
					# (i.e., field width) of a printf-style format (not always right)
					function formatLength(format) {
						fmtLen = format
						sub(/%[^0-9]*/, "", fmtLen)
						sub(/[^0-9].*$/, "", fmtLen)
						return (fmtLen + 0)
					}
					BEGIN {
						# load some shell variables into awk
						doSpaceFilling = "'"$DO_SPACE_FILLING"'" + 0

						spaces = "                         "  # space-filling spaces
						spaces = (spaces)(spaces)(spaces)(spaces)(spaces)
					}
					(NR == 1) {
						numResultFields = "'"$NUM_RESULT_CSV_FIELDS"'" + 0
						fieldFormats = "'"$FIELD_FORMAT_LIST"'"

						# create an array of field formats
						split(fieldFormats, theFieldFormat, ",")

						# ensure there is a field format for all fields
						for (i = 1; i <= numResultFields; i++) {
								if (length(theFieldFormat[i]) == 0) {
									theFieldFormat[i] = "%s"
								}
						}

						# output the header record
						print $0
					}
					(NR > 1) {
						fieldSep = ""  # empty field separator for first field

						# output a record
						for (i = 1; i <= numResultFields; i++) {  # output each field
							if ((length($i) != 0) && (match($i, /[^ ]/) != 0)) {
								# if this is not a value that is only whitespace, output
								# the formatted value
								printf("%s"theFieldFormat[i], fieldSep, $i)
							}
							else {
								# field is empty or is only whitespace
								# - if not doing space-filling, output an empty field
								# - otherwise, output a format-width of spaces
								if (doSpaceFilling == 1) {
									printf("%s%s", fieldSep, \
									  substr(spaces, 1, formatLength(theFieldFormat[i])))
								}
								else { printf("%s", fieldSep) }
							}

							fieldSep = ","
						}
						printf("\n")
					}
				'"'"
elif test "$FORMULAS_ARE_USED" = '1'
	then
		# formulas are being used but no field-formats are being used so process
		# the formulas via bc then do some cleanup processing
		fCMD='# process the output using the bc utility to "do the math"
				/usr/bin/bc -l | \
				# get rid of the newlines added by bc
				/usr/bin/tr -d "\n" | \
				# translate the bcEOF-generated tilde field separators into commas
				/usr/bin/tr "~" "," | \
				# translate the added ? EOL characters into newlines
				/usr/bin/tr "?" "\n" | \
				# get rid of the backslash characters added by bc
				/usr/bin/tr -d "\\" | \
				# remove empty line at the end of the file
				/usr/bin/sed -e "/^[ \	]*$/d"'
elif test "$FIELD_FORMATS_ARE_USED" = '1'
then
	# with no formulas, field formatting is done during the "major" processing
	fCMD='/bin/cat'
else
	# no formulas or formatting -- just do a no-change pass-through
	fCMD='/bin/cat'
fi
#echo "\nfCMD=$fCMD"

# if using formulas, escape any double-quotes in the header record
if test "$FORMULAS_ARE_USED" = '1'
then
	THE_RESULT_CSV_FIELD_NAMES=`
				echo "$RESULT_CSV_FIELD_NAMES" | /usr/bin/sed -e 's/"/\\\\\\\\q/g'`
else
	THE_RESULT_CSV_FIELD_NAMES="$RESULT_CSV_FIELD_NAMES"
fi
#echo "\nRESULT_CSV_FIELD_NAMES = $RESULT_CSV_FIELD_NAMES"
#echo "\nTHE_RESULT_CSV_FIELD_NAMES = $THE_RESULT_CSV_FIELD_NAMES"

# create the "work file":
# - unless showing selected fields/records is enabled, ignore the header record
# - ignore any records having an invalid field count
# - include only the the selected source CSV fields
# - trim surrounding whitespace from the field values
# - if formulas are bing used, remove all double-quotes from data records
/usr/bin/awk -F ',' -v theResultCSVfieldNames="$THE_RESULT_CSV_FIELD_NAMES" '
	# defines leading/trailing whitespace-trimming functions
	# trim() may be used by the HEADER_PRINT_CMD and DATA_PRINT_CMD
	function ltrim(s) { sub(/^[ \t]+/, "", s); return s }
	function rtrim(s) { sub(/[ \t]+$/, "", s); return s }
	function trim(s) { return rtrim(ltrim(s)); }

	# defines a function to remove double-quote characters (PRINT_CMD may use it)
	function deQuote(s) { gsub(/\"/, "", s) ; return s }

	(NR == 1) {
		# load some shell variables into awk
		showSelectedFields = "'"$SHOW_SELECTED_FIELDS"'" + 0
		joinFields = "'"$JOIN_FIELDS"'" + 0

		# create an array of field names to get the number of fields in the source
		# CSV file ... assumes the 1st record has the correct number of fields
		numSourceCSVfields = split($0, sourceFieldName, ",")

		# initialize some needed variables/constants
		invalidRecsSkipped = 0

		# if required, output the header record
		if (showSelectedFields == 1) { printf("%s\n", theResultCSVfieldNames) }
		if (joinFields == 1) { print '"$HEADER_PRINT_CMD"' }
	}
	(NR > 1) {
		# skip any records that do not have the correct number of fields and
		# extract/output the applicable fields from those records that do
		if (NF != numSourceCSVfields) {
			invalidRecsSkipped++
		}
		else {
			# output the record
			print '"$DATA_PRINT_CMD"'
		}
	}
	END {
		# output any non-zero counts information
		if (invalidRecsSkipped > 0) {
			system("echo >&2 ; echo NOTE: ignored "sprintf("%\047d", invalidRecsSkipped)" data record\\(s\\) that had an incorrect number of fields >&2")
		}
	}' "$WORK_FILE_PATHNAME" | \
# if required, sort the records using the supplied sort fields
eval "$sCMD" | \
# --------------------------------------
# perform the "major" processing ...
# as applicable, process the valid extracted/sorted records as specified:
# - transpose field names, in the header record
# - fill empty fields
# - perform required fields selection
# - apply formula processing
# - format the field values
/usr/bin/awk -F ',' -v theFormulasSpec="$FULL_FORMULAS_SPEC_AS_LINE" \
						  -v theResultCSVfieldNames="$THE_RESULT_CSV_FIELD_NAMES" '
	# defines leading/trailing whitespace-trimming functions
	function ltrim(s) { sub(/^[ \t]+/, "", s); return s }
	function rtrim(s) { sub(/[ \t]+$/, "", s); return s }
	function trim(s) { return rtrim(ltrim(s)); }

	# defines a function that tries to determine the output length (i.e., field
	# width) of a printf-style format (not always correct)
	function formatLength(format) {
		fmtLen = format
		sub(/%[^0-9]*/, "", fmtLen)
		sub(/[^0-9].*$/, "", fmtLen)
		return (fmtLen + 0)
	}

	# adapted from: http://howardhinnant.github.io/date_algorithms.html (Thanks!)
	# get the date/time as "YYYY-MM-DD HH:MM:SS[.sss]" from a UNIX-style UTC
	# "seconds or milliseconds from epoch" number (en) where:
	# - hasMsec optionally indicates that the epoch number includes milliseconds
	# - tzo optionally supplies the time-zone offset from UTC in seconds which is
	#   used to adjust the result date/time
	#  (UNIX-style epoch = 1970-01-01 00:00:00[.000])
	#
	function getDateTime(en, hasMsec, tzo) {
		# if applicable, retrieve and remove the msec from the epoch number
		if ((length(hasMsec) != 0) && (hasMsec == 1)) {
			msec = "."(sprintf("%03d", (en % 1000)))
			en = int(en/1000)
		}
		else { msec = "" }

		if (length(tzo) != 0) { en = en + tzo } #if required, add time-zone offset

		# retrieve and remove the seconds, minutes and hours from the epoch number
		sec = en % 60
		en = int(en/60)
		min = en % 60
		en = int(en/60)
		hr = en % 24
		en = int(en/24)   # epoch number is now "days from epoch"
		en = en + 719468  # bias epoch base to UNIX epoch of 1970-01-01

		# Gregorian calendar repeats every 400 yr (each "era")
		# get the era in which this epoch number exists
		if (en >= 0) { era = en } else { era = en - 146096 }
		era = int(era/146097)

		# get the day and year of/within the era -- ranges [0,146096] and [0,399]
		doe = en - (era * 146097)
		yoe = int((doe - int(doe/1460) + int(doe/36524) - int(doe/146096))/365)

		y = yoe + (era * 400)  # year
		doy = doe - ((365 * yoe) + int(yoe/4) - int(yoe/100))  # day of yr [0,365]
		mp = int(((5 * int(doy)) + 2)/153)  # month position (0=Mar,11=Feb) [0,11]
		d = doy - int(((153 * mp) + 2)/5) + 1  # day [1,31]

		# from the month position, get the month and, if required, adjust the year
		if (mp < 10) { m = mp + 3 } else { m = mp - 9; y = y + 1 }  # [1, 12]

			return sprintf("%04d-%02d-%02d %02d:%02d:%02d%s", \
																	y, m, d, hr, min, sec, msec)
	}

	# adapted from: http://howardhinnant.github.io/date_algorithms.html (Thanks!)
	# get the UNIX-style UTC "seconds or milliseconds from epoch" number from a
	# date/time "timestamp" (ts) in the input format (fmt) -- if the date/time is
	# in local time, the resulting epoch number can be adjusted to UTC by
	# providing the time-zone offset (tzo) from UTC in seconds
	# (UNIX-style epoch = 1970-01-01 00:00:00[.000])
	#
	# the supported date/time input formats (fmt) are (default is format 1):
	# 1 YMD HH:MM:SS : e.g., Y-M-D H:M:S or YY/MM/DDTHH:MM:SS.sss
	# 2 DMY HH:MM:SS : e.g., D-M-Y H:M:S[.sss] or DD/MMM/YY HH:MM:SS
	# 3 MDY HH:MM:SS : e.g., M-D-Y H:M:S[.sss] or MMM/DD/YY HH:MM:SS
	#
	# - where: Y = year, M = month, D = day and
	#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
	# - M, D, H, M and S can be 1 or 2 digits
	# - M can also be the month-name abbreviations consisting of the first 3
	#   characters of the English month name, Jan-Dec (not case sensitive)
	# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
	#   is added to Y (e.g., Y=123 --> 2123, Y=10 --> 2010) ...  2-digit years,
	#   because the century is removed, are valid only for the current century
	# - date separators can be either - or /
	# - time is HH:MM:SS[.sss]  (where [.sss] means "with or without msec")
	# - the date/time separator can be either a space or T (not case sensitive)
	#
	# NOTE: depends upon the awk global variables: currCentury and monthNum
	#
	function getEpochNum(ts, fmt, tzo) {
		# if it is present, replace the T date/time separator with a space
		p = match(ts, /[Tt][0-9][0-9]*\:/)
		if (p != 0) { ts = substr(ts, 1, (p - 1))" "substr(ts, (p + 1)) }

		# NOTE: the "split by character" will fail if FS includes that character!
		split(ts, a, /[\-:\.\/ ]/)
		#for (item in a){ printf("a[%s] = %s, item = %s\n", item, a[item], item) }
		#for (item in a){ system("echo a\\[\""item"\"\\] = \""a[item]"\" >^2") }

		# based upon the format, extract the year, month and day
		if ((length(fmt) == 0) || (fmt == 1)) {  # default is format # 1 (YMD)
			y = a[1] + 0
			m = a[2]
			d = a[3] + 0
		}
		else if (fmt == 2) {  # DMY
			y = a[3] + 0
			m = a[2]
			d = a[1] + 0
		}
		else if (fmt == 3) {  # MDY
			y = a[3] + 0
			m = a[1]
			d = a[2] + 0
		}
		else { return 0 }

		# extract the hour, minute, seconds and optional milliseconds
		hr = a[4] + 0
		min = a[5] + 0
		sec = a[6] + 0
		if (length(a[7]) != 0) { msec = a[7] + 0 }  # assign msec, if available

		# if it is not a 4-digit year, determine the year
		if (length(y) != 4) { y = y + currCentury }

		# if not a numeric month, use the 3-char month to lookup the month number
		if (match(m, /[0-9]/) == 0) { m =  monthNum[tolower(m)] + 0 }
		else { m = m + 0 }
		#printf("%d-%02d-%02d %02d:%02d:%02d\n", y, m, d, hr, min, sec)

		if (length(tzo) == 0) { tzo = 0 }
		if (m <= 2) { y = y - 1 }

		# Gregorian calendar repeats every 400 yr (each "era")
		# get the era in which this date exists
		if (y >= 0) { era = y } else { era = y - 399 }
		era = int(era/400)

		yoe = y - (era * 400)  # get the year of/within the era, range [0,399]

		# get the day of year, range [0,365]
		if (m > 2) { doy = int(((153 * (m-3)) + 2)/5) + d-1 }
		else { doy = int(((153 * (m + 9)) + 2)/5) + d-1 }

		doe = (yoe * 365) + int(yoe/4) - int(yoe/100) + doy #day of era [0,146096]

		# get the "days from epoch" number biased to UNIX epoch of 1970-01-01
		en = (era * 146097) + int(doe) - 719468

		# compute "seconds from epoch" by adding the hours, minutes and seconds
		# and subtracting the time-zone offset seconds (i.e., the supplied
		# date/time was in local time, UTC + time-zone offset, to the time-zone
		# offset is subtracted from the epoch number to move it to UTC)
		en = (en * 86400) + (3600 * hr) + (60 * min) + sec - tzo

		# if applicable, add the milliseconds
		if (length(a[7]) != 0) { en = (en * 1000) + msec }

		return en
	}

	# transpose a date/time "timestamp" (ts) from the input format (inFmt) to the
	# output format (outFmt) with milliseconds appended if inclMs = 1 and without
	# leading zeros for M, D, Y, H, M, S if shortForm = 1
	#
	# the supported date/time input formats (fmt) are (default is format 1):
	# 1 YMD HH:MM:SS : e.g., Y-M-D H:M:S or YY/MM/DDTHH:MM:SS.sss
	# 2 DMY HH:MM:SS : e.g., D-M-Y H:M:S[.sss] or DD/MMM/YY HH:MM:SS
	# 3 MDY HH:MM:SS : e.g., M-D-Y H:M:S[.sss] or MMM/DD/YY HH:MM:SS
	# - where: Y = year, M = month, D = day and
	#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
	# - M, D, H, M and S can be 1 or 2 digits
	# - M can also be the month-name abbreviations consisting of the first 3
	#   characters of the English month name, Jan-Dec (not case sensitive)
	# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
	#   is added to Y (e.g., Y=123 --> 2123, Y=10 --> 2010)
	# - date separators can be either - or /
	# - time is HH:MM:SS[.sss]  (where [.sss] means "with or without msec")
	# - date/time separator can be either a space or T (not case sensitive)
	#
	# the supported date/time output formats (outFmt) are (default is format 1):
	# YMD formats ...
	#  1 YYYY-MM-DD HH:MM:SS.sss
	#  2 YYYY-mmm-DD HH:MM:SS.sss
	#  3 YY-MM-DD HH:MM:SS.sss
	#  4 YY-mmm-DD HH:MM:SS.sss
	# 
	#  5 YYYY/MM/DD HH:MM:SS.sss
	#  6 YYYY/mmm/DD HH:MM:SS.sss
	#  7 YY/MM/DD HH:MM:SS.sss
	#  8 YY/mmm/DD HH:MM:SS.sss
	# 
	# DMY formats ...
	#  9 DD-MM-YYYY HH:MM:SS.sss
	# 10 DD-mmm-YYYY HH:MM:SS.sss
	# 11 DD-MM-YY HH:MM:SS.sss
	# 12 DD-mmm-YY HH:MM:SS.sss
	# 
	# 13 DD/MM/YYYY HH:MM:SS.sss
	# 14 DD/mmm/YYYY HH:MM:SS.sss
	# 15 DD/MM/YY HH:MM:SS.sss
	# 16 DD/mmm/YY HH:MM:SS.sss
	# 
	# MDY formats ...
	# 17 MM-DD-YYYY HH:MM:SS.sss
	# 18 mmm-DD-YYYY HH:MM:SS.sss
	# 19 MM-DD-YY HH:MM:SS.sss
	# 20 mmm-DD-YY HH:MM:SS.sss
	# 
	# 21 MM/DD/YYYY HH:MM:SS.sss
	# 22 mmm/DD/YYYY HH:MM:SS.sss
	# 23 MM/DD/YY HH:MM:SS.sss
	# 24 mmm/DD/YY HH:MM:SS.sss
	# 
	# - where: Y = year, M = month, D = day and
	#          HH = hours, MM = minutes, SS = seconds and .sss = optional msec
	# - M and D  1 or 2 digits
	# - H, M and S are always 2 digits and the optional .sss is always 3 digits
	# - mmm is a month-name abbreviation consisting of the first 3 characters
	#   of the English name for the month, Jan-Dec (not case sensitive)
	# - Y can be 1 to 4 digits -- if Y < 4 digits, then the current century
	#   is added to Y (e.g., Y=123 --> 2123, Y=10 --> 2010)
	# - date separators can be either - or /
	# - time is HH:MM:SS.sss  (where .sss means "with or without msec")
	# - date/time separator can be either a space or T (not case sensitive)
	#
	# NOTE: depends upon awk global variables: currCentury, monthNum & monthName
	#
	function transposeDateTime(ts, inFmt, outFmt, inclMs, shortForm) {
		# if it is present, replace the T date/time separator with a space
		p = match(ts, /[Tt][0-9][0-9]*\:/)
		if (p != 0) { ts = substr(ts, 1, (p - 1))" "substr(ts, (p + 1)) }

		# NOTE: the "split by character" will fail if FS includes that character!
		split(ts, a, /[\-:\.\/ ]/)
		#for (item in a){ printf("a[%s] = %s, item = %s\n", item, a[item], item) }
		#for (item in a){ system("echo a\\[\""item"\"\\] = \""a[item]"\" >^2") }

		# based upon the format, extract the year, month and day
		if ((length(inFmt) == 0) || (inFmt == 1)) {  # default is format # 1 (YMD)
			y = a[1] + 0
			m = a[2]
			d = a[3] + 0
		}
		else if (inFmt == 2) {  # DMY
			y = a[3] + 0
			m = a[2]
			d = a[1] + 0
		}
		else if (inFmt == 3) {  # MDY
			y = a[3] + 0
			m = a[1]
			d = a[2] + 0
		}
		else { return "bad input format" }

		# extract the hour, minute, seconds and optional milliseconds
		hr = a[4] + 0
		min = a[5] + 0
		sec = a[6] + 0

		# get msec string
		if ((length(inclMs) != 0) && (inclMs == 1)) {
			if (length(a[7]) == 0) { msec = ".000" }
			else { msec = "."(sprintf("%03d", a[7])) }
		}
		else { msec = "" }

		# determine the 4-digit year & the output-formatting for the 4-digit year
		if (length(y) != 4) { y = y + currCentury }  # add century -> 4-digit year
		yFmt = "%4d"

		# if required, determine the 2-digit year and the output-formatting for it
		if ((outFmt == 3) || (outFmt == 4) || (outFmt == 7) || (outFmt == 8) || 
			 (outFmt == 11) || (outFmt == 12) || (outFmt == 15) || (outFmt == 16) ||
			 (outFmt == 19) || (outFmt == 20) || (outFmt == 23) || (outFmt == 24)) {
			y = y % 100  # the year in the century (i.e., the 2-digit year)
			yFmt = "%02d"
			if (shortForm == 1) { yFmt = "%d" }
		}

		# if output format (is an odd number and so) requires a numeric month and
		# it is not a numeric month, use the 3-char month to lookup month number
		if ((outFmt % 2) == 1) {  # odd-numbered output format
			mFmt = "%02d"
			if (shortForm == 1) { mFmt = "%d" }
			if (match(m, /[0-9]/) == 0) { m =  monthNum[tolower(m)] + 0 }
			else { m = m + 0 }
		}

		# if output format (is an even number and so) requires a 3-character month
		# and it is a numeric month, use the month number to lookup 3-char month
		if ((outFmt % 2) == 0) {  # even-numbered output format
			mFmt = "%s"
			if (match(m, /[0-9]/) != 0) { m =  monthName[m+0] }
		}
		#printf("%d-"mFmt"-%"nd"d %"nd"d:%"nd"d:%"nd"d%s\n", \
		#															y, m, d, hr, min, sec, msec)

		# set the day-of-month numeric format
		if ((length(shortForm) != 0) && (shortForm == 1)) { dFmt = "%d" }
		else { dFmt = "%02d" }

		# set the YMD separator character
		sep = "-"
		if (((outFmt >= 5) && (outFmt <= 8)) ||
			 ((outFmt >= 13) && (outFmt <= 16)) ||
			 ((outFmt >= 21) && (outFmt <= 24))) { sep = "/" }

		# output the date/time
		if ((outFmt >= 1) && (outFmt <= 8)) {  # output in YMD format
			return sprintf(yFmt""sep""mFmt""sep""dFmt" %02d:%02d:%02d%s",
																	y, m, d, hr, min, sec, msec)
		}
		else if ((outFmt >= 9) && (outFmt <= 16)) {  # output in DMY format
			return sprintf(dFmt""sep""mFmt""sep""yFmt" %02d:%02d:%02d%s",
																	d, m, y, hr, min, sec, msec)
		}
		else if ((outFmt >= 17) && (outFmt <= 24)) {  # output in MDY format
			return sprintf(mFmt""sep""dFmt""sep""yFmt" %02d:%02d:%02d%s",
																	m, d, y, hr, min, sec, msec)
		}
		else { return "bad output format" }
	}
	BEGIN {
		# exit if showing selected fields/records
		if (("'"$SHOW_SELECTED_FIELDS"'" + 0) == 1) { exit }

		# load some shell variables into awk
		numResultFields = "'"$NUM_RESULT_CSV_FIELDS"'" + 0
		joinFields = "'"$JOIN_FIELDS"'" + 0
		doTrim = "'"$DO_TRIM"'" + 0
		tzOffsetSec = "'"$TZ_OFFSET_SEC"'" + 0
		fillEmptyFields = "'"$FILL_EMPTY_FIELDS"'" + 0
		doSpaceFilling = "'"$DO_SPACE_FILLING"'" + 0
		fieldFormatsAreUsed = "'"$FIELD_FORMATS_ARE_USED"'" + 0
		formulasAreUsed = "'"$FORMULAS_ARE_USED"'" + 0
		resultsAsFormulas = "'"$RESULTS_AS_FORMULAS"'" + 0
		requiredFields = "'"$REQUIRED_FIELDS_FLAGS"'"
		fieldFormats = "'"$FIELD_FORMAT_LIST"'"
		appliedFormulaNums = "'"$FORMULA_NUM_LIST"'"
		currYear = "'"$CURR_YR_NUM"'" + 0

		# initialize some needed variables/constants
		spaces = "                         "  # spaces for space-filling
		spaces = (spaces)(spaces)(spaces)(spaces)(spaces)
		numRecordsOutput = 0
		initialRecsSkipped = 0
		recsMissingRequiredFieldSkipped = 0
		formulaAppliedToNonNumeric=0
		rdIsUsed = 0
		b4IsUsed = 0
		tzoIsUsed = 0
		prior = ""  # holds the initialized b4 "previous value" array used in bc
		scaleStmt = "scale=20;"  # the default scale at the start of each formula
		builtInFnList="eNumToUTCdateTime,eNumToLocalDateTime,eNumToUTCdateTimeMsec,eNumToLocalDateTimeMsec,UTCdateTimeYMDtoEnum,localDateTimeYMDtoEnum,UTCdateTimeDMYtoEnum,localDateTimeDMYtoEnum,UTCdateTimeMDYtoEnum,localDateTimeMDYtoEnum,tDT"
		currCentury = currYear - currYear % 100

		# create "month-name to month-number and "month-number to month-name"
		# lookup tables (associative arrays)
		monthNames="Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec"
		split(monthNames, month, ",")
		for (i = 1; i <= 12; i++) { monthNum[tolower(month[i])] = i }
		for (i = 1; i <= 12; i++) { monthName[i] = month[i] }
		#for (it in monthNum) {printf("monthNum[%s] = %s\n", it, monthNum[it])}
		#for (it in monthName) {printf("monthName[%s] = %s\n", it, monthName[it])}

		# a tilde, enclosed in double-quotes, is the "formula" to have bc
		# output a tilde, as our end-of-field separator (later translated to ",")
		bcEOF = "\"~\";   "

		# create arrays to hold the current and previous values
		for (i = 1; i <= numResultFields; i++) {
			currVal[i] = ""
			prevVal[i] = ""
		}

		# create an array of built-in functions
		numBuiltInFns = split(builtInFnList, builtInFn, ",")

		# create an array of required-fields flags/indicators
		split(requiredFields, isRequiredField, ",")

		# ensure there is a required-fields flag for all fields
		for (i = 1; i <= numResultFields; i++) {
			if ((length(isRequiredField[i]) == 0) ||
				 (isRequiredField[i] != 1)) { isRequiredField[i] = 0 }
		}

		# create an array of field formats
		split(fieldFormats, theFieldFormat, ",")

		# ensure there is a field format for all fields
		for (i = 1; i <= numResultFields; i++) {
			if (length(theFieldFormat[i]) == 0) { theFieldFormat[i] = "%s" }
		}

		# create an array of formula entries
		numFormulaEntries = split(theFormulasSpec, formulaEntry, "?")

		# create an associative array to enable a formula to be looked up
		# (i.e., indexed) by its formula ID number so:
		# aFormula[<ID number>] is the <formula>
		for (i = 1; i <= numFormulaEntries; i++) {
			split(formulaEntry[i], IDandFormula, "~")  # splits -> ID & formula
			aFormula[sprintf("%d", IDandFormula[1])] = IDandFormula[3] #ID->formula
		}

		# create an array of the formula numbers to be applied
		split(appliedFormulaNums, appliedFormulaNum, ",")

		# ensure there is a valid applied formula number for each field (0 = none)
		# and determine whether the rounding function is used and, for each
		# formula, whether it uses the b4() "pseudo-function"
		for (i = 1; i <= numResultFields; i++) {
			formulaUSESb4[i] = 0

			if (length(appliedFormulaNum[i]) == 0) { appliedFormulaNum[i] = 0 }
			else if (length(aFormula[appliedFormulaNum[i]]) == 0) {
				if (appliedFormulaNum[i] != 0) {
					system("echo >&2 ; echo WARNING: formula number "appliedFormulaNum[i]" is invalid and was ignored >&2")
				}
				appliedFormulaNum[i] = 0
			}
			else {
				# have a legitimate formula number
				theFormula = aFormula[appliedFormulaNum[i]]

				if (index(theFormula, "rd(") != 0) { rdIsUsed = 1 }

				if (index(theFormula, "b4(") != 0) {
					formulaUSESb4[i] = 1
					b4IsUsed = 1
				}

				if (index(theFormula, "tzo") != 0) { tzoIsUsed = 1 }
			}
		}

		# define the rounding function used in bc
		rdFn = "define rd(n,decs){auto d,f,x,z;d=decs;x=0;z=scale;scale=d+1;if(n<0){x=1;n=-(n)};f=n+(5/10^(d+1));scale=d;f=(f*(10^d))/(10^d);if(x>0){f=-(f)};scale=z;return(f)};"

		# create (the string to create) an initialized array that will be used in
		# bc to hold the prior field value that was calculated via bc using the
		# applied formula
		for (i = 1; i <= numResultFields; i++) {
			priorArray = (priorArray)sprintf("prior[%d]=0;", i)
		}

		# if outputting the formulas, also output some "how to read" instructions
		if ((formulasAreUsed == 1) && (resultsAsFormulas == 1)) {
			print "How to read this output:"
			print "- the following output is piped through \"bc -l\" as a continuous stream"
			print "- statements (assignments, formulas, etc.) are terminated via a semi-colon"
			print "- characters surrounded by double-quotes are \"passed through as text\""
			print "- the sequence  \"~\";  will later become the comma field-separators in the result CSV file"
			print "- the sequence  \"?\"  will later become the record-terminators (EOLs) in the result CSV file"
			print "- scale is (re)set at the end of each field for use at the beginning of the next field"
			print "-------------------------------------------------------------------------------------------"
		}

		if (formulasAreUsed == 1) {
		# formulas are being applied:
		# - set bcEOF as the field separator
		# - output the (bc code to print the) header record
		# - if required, output the rounding function
		# - output the (default) scale-setting statement
		# - if required, output prior-value array and/or timezone offset variable
			fieldSep = bcEOF
			printf("print \"%s\"; \"?\"\n", theResultCSVfieldNames)
			if (rdIsUsed == 1) { printf("%s\n", rdFn) }
			printf("%s", scaleStmt)
			if (tzoIsUsed == 1) { printf(" tzo=%d;", tzOffsetSec) }
			if (b4IsUsed == 1) { printf(" %s", priorArray) }
			printf("\n")
		}
		else {
			# no formulas are being applied:
			# - set a comma as the field-separator
			# output the header record
			fieldSep = ","
			if (joinFields != 1) { printf("%s\n", theResultCSVfieldNames) }
		}
	}
	{
		# clear various values
		recOut = ""
		fieldSeparator = ""  # empty field separator for first field
		isMissingRequiredField = 0
		noInitialValue = 0
		recIsSkipped = 0
		formulaHasCurr = 0
		formulaHasPrev = 0

		# ignore record if it is missing a value for one or more required fields
		if (length(requiredFields) != 0) {
			# determine whether the record is missing any required fields
			for (i = 1; i <= numResultFields; i++) {
				if ((isRequiredField[i] == 1) && (length($i) == 0)) {  # is missing!
					isMissingRequiredField = 1
					recsMissingRequiredFieldSkipped++
					break
				}
			}
		}

		# process all the fields in a record
		for (i = 1; i <= numResultFields; i++) {
			# clear some variables
			formulaHasCurr = 0
			formulaHasPrev = 0
			priorVal = ""

			# add a field separator to the output (fs is empty for the first field)
			recOut = (recOut)(fieldSeparator)

			# ensure the field separator is not empty after first field
			fieldSeparator = fieldSep

			# ignore this record if it is missing a value for a required field
			if (isMissingRequiredField == 1) {
				recIsSkipped = 1

				# ensure record still participates in field-filling
				if (fillEmptyFields == 1) { continue }

				break  # this record is ignored, entirely
			}

			# if not filling empty fields and not applying any formulas, arrange to
			# output the entire record
			if ((fillEmptyFields != 1) &&
				 (formulasAreUsed != 1) && (fieldFormatsAreUsed != 1)) {
				recOut = $0
				break
			}

			# set the current value to the value of this field
			currVal[i] = $i

			# if required, create the prior[] variable for this field (used by bc)
			if ((formulasAreUsed == 1) && (formulaUSESb4[i] == 1)) {
				priorVal = sprintf("prior[%d]", i)
			}

			# if empty fields are to be filled, then there can be no empty fields,
			# so skip any initial entries that have empty fields and no previous
			# value has yet been set -- i.e., if there is no previous value for
			# this field and there is no current value for this field, this
			# record is ignored because the CSV file cannot have empty fields
			if ((fillEmptyFields == 1) &&
				 (length(prevVal[i]) == 0) && (length(currVal[i]) == 0)) {
				noInitialValue = 1
				recIsSkipped = 1
				continue  # do not break, still need to pick up missing field values
			}

			# if the previous value for this field is not yet set and there is a
			# current value for this field, set the previous value to this
			# current value (i.e., first prevVal is the first-encountered currVal)
			if ((length(prevVal[i]) == 0) && (length(currVal[i]) != 0)) {
				prevVal[i] = currVal[i]

				# if this record will not be skipped and the b4() "pseudo function"
				# is used (i.e., so the prior value will be used, in bc), the set
				# the prior value to this current value (i.e., the first prior value
				# is the first-encountered currVal)
				if ((recIsSkipped != 1) &&
					 (formulasAreUsed == 1) && (formulaUSESb4[i] == 1)) {
					recOut = (recOut)sprintf(" %s=%s;", priorVal, currVal[i])
				}
			}

			# if filling empty fields via "copy forward" and if there is NO current
			# value for this field and the previous value for this field is set,
			# then set the current value for this field to the previous value
			if ((fillEmptyFields == 1) &&
				 (length(currVal[i]) == 0) && (length(prevVal[i]) != 0)) {
				currVal[i] = prevVal[i]
			}

			# if this record is to be skipped, there is no need to continue
			# processing this field
			if (recIsSkipped == 1) { continue }

			# determine the formula and whether it uses curr and/or prev variables
			if (formulasAreUsed == 1) {
				# if available, get the formula for this field
				formula = aFormula[appliedFormulaNum[i]]

				if (length(formula) != 0) {
					# determine whether formula needs curr and/or prev variables
					if ((length(currVal[i]) != 0) &&
						 (index(formula, "curr") != 0)) { formulaHasCurr = 1 }

					if ((length(prevVal[i]) != 0) &&
						 (index(formula, "prev") != 0)) { formulaHasPrev = 1 }

					# ensure the formula has a semi-colon terminator
					if (substr(formula, length(formula), 1) != ";") {
						formula = (formula)";"
					}
				}
			}

			# if required, apply the formula/computation to this field - i.e., if:
			# - formulas are being used/applied AND
			# - there is a formula for this field
			# - the current field value is not empty AND
			if ((formulasAreUsed == 1) && (length(formula) != 0) &&
				 (length(trim(currVal[i])) != 0)) {
				# ensure whitespace is removed before/after the field value
				if (doTrim == 0) { currVal[i] = trim(currVal[i]) }

				# if required, perform the built-in function
				for (j = 1; j <= numBuiltInFns; j++) {
					# attempt to find a built-in formula via the builtInFnList map
					if (index(formula, (builtInFn[j])"(") != 0) {
						# apply the buil-in formula where f is field == currVal[i]
						# j=1 is eNumToUTCdateTime       --> getDateTime(f)
						# j=2 is eNumToLocalDateTime     --> getDateTime(f, 0, tzo)
						# j=3 is eNumToUTCdateTimeMsec   --> getDateTime(f, 1)
						# j=4 is eNumToLocalDateTimeMsec --> getDateTime(f, 1, tzo)
						# j=5 is UTCdateTimeYMDtoEnum    --> getEpochNum(f, 1)
						# j=6 is localDateTimeYMDtoEnum  --> getEpochNum(f, 1, tzo)
						# j=7 is UTCdateTimeDMYtoEnum    --> getEpochNum(f, 2)
						# j=8 is localDateTimeDMYtoEnum  --> getEpochNum(f, 2, tzo)
						# j=9 is UTCdateTimeDMYtoEnum    --> getEpochNum(f, 2)
						# j=10 is localDateTimeDMYtoEnum --> getEpochNum(f, 2, tzo)
						# j=11 is tDT --> transposeDateTime(f, inFmt, outFmt, inclMs, shortForm)
						if (j == 1) {
							recOut = (recOut)sprintf("\"%s\";", \
																		getDateTime(currVal[i]))
							formula = ""
						}
						else if (j == 2) {
							recOut = (recOut)sprintf("\"%s\";", \
													getDateTime(currVal[i], 0, tzOffsetSec))
							formula = ""
						}
						else if (j == 3) {
							recOut = (recOut)sprintf("\"%s\";", \
																	getDateTime(currVal[i], 1))
							formula = ""
						}
						else if (j == 4) {
							recOut = (recOut)sprintf("\"%s\";", \
													getDateTime(currVal[i], 1, tzOffsetSec))
							formula = ""
						}
						else if (j == 5) {
							recOut = (recOut)sprintf("\"%s\";", \
																	getEpochNum(currVal[i], 1))
							formula = ""
						}
						else if (j == 6) {
							recOut = (recOut)sprintf("\"%s\";", \
													getEpochNum(currVal[i], 1, tzOffsetSec))
							formula = ""
						}
						else if (j == 7) {
							recOut = (recOut)sprintf("\"%s\";", \
																	getEpochNum(currVal[i], 2))
							formula = ""
						}
						else if (j == 8) {
							recOut = (recOut)sprintf("\"%s\";", \
													getEpochNum(currVal[i], 2, tzOffsetSec))
							formula = ""
						}
						else if (j == 9) {
							recOut = (recOut)sprintf("\"%s\";", \
																	getEpochNum(currVal[i], 3))
							formula = ""
						}
						else if (j == 10) {
							recOut = (recOut)sprintf("\"%s\";", \
													getEpochNum(currVal[i], 3, tzOffsetSec))
							formula = ""
						}
						else if (j == 11) {
							# remove the function name and opening parenthesis from the
							# formula
							# (do not enclose re in slashes if variable is the pattern)
							sub((builtInFn[j])"\\(", "", formula)

							# remove the closing paren and any subsequent characters
							# from the formula
							sub("\\).*", "", formula)

							# extract the function parameters
							split(formula, arg, ",")

							recOut = (recOut)sprintf("\"%s\";", \
											transposeDateTime(currVal[i], arg[1], arg[2], \
																					arg[3], arg[4]))
							formula = ""
						}

						break
					}
				}

				# with these built-in operations, no additional formula can be
				# applied to this field (formula is empty after a built-in formula)
				if (length(formula) == 0) { continue }

				# if this field is not a number, it cannot be assigned to a bc
				# variable -- output it "as is"
				if(match(currVal[i], /[^0-9\-+\.]/) != 0) {  # it is not a number
					recOut = (recOut)sprintf("\"%s\"; %s", currVal[i], scaleStmt)
					formulaAppliedToNonNumeric = 1
					continue
				}

				# if required, add the current/previous field-value assignments
				if (formulaHasCurr == 1) {
					recOut = (recOut)sprintf("curr=%s;", currVal[i])
				}
				if (formulaHasPrev == 1) {
					recOut = (recOut)sprintf("prev=%s;", prevVal[i])
				}

				# if required, modify the formula and add the prior field-value
				# assignment
				if (formulaUSESb4[i] == 1) {
					# change the b4() pseudo-function to a prior[] value assignment --
					# this changes: b4(<expression>) to prior[i]=(<expression>)
					# so the computed field value will be set as the prior value for
					# this field
					sub(/b4\(/, (priorVal)"=(", formula)

					# add the prior field-value assignment -- i.e., set the prior var
					recOut = (recOut)"prior="(priorVal)";"
				}

				# if still applicable, add the (possibly modified) formula to output
				if (length(formula) != 0) {recOut = (recOut)sprintf(" %s", formula)}

				# if required, add the prior output of the computed value -- i.e.,
				# if the prior[i] variable was set to the value of the bc-computed
				# field, by adding that variable to the output, the current value
				# will be output and prior[i] is "all set" for the next record
				if (formulaUSESb4[i] == 1) {
					recOut = (recOut)sprintf(" %s;", priorVal)
				}

				# output the scale-setting statement to set scale for next formula
				recOut = (recOut)" "(scaleStmt)
			}
			else if (formulasAreUsed == 1) {
				# when formulas are being used but the current field requires no
				# formula to be applied, output the field value enclosed in double
				# quotes followed by a semi-colon (statement separator) so the bc
				# utility  will output those values, minus the double quotes and
				# semi-colon (effectively, a field-value pass-through)
				recOut = (recOut)sprintf("\"%s\"; %s", currVal[i], scaleStmt)
			}
			else {
				# there are no formulas to be applied, if this is not a value that
				# is only whitespace, just add the data-field value to the output
				# and apply the specified field formatting (%s if nothing spec"d)
				if ((length(currVal[i]) != 0) && (match(currVal[i], /[^ ]/) != 0)) {
					recOut = (recOut)sprintf(theFieldFormat[i], currVal[i])
				}
				else {
					# field is empty or is only whitespace
					# - if doing space-filling, output a format-width of spaces
					# - otherwise, output an empty field
					if (doSpaceFilling == 1) {
						recOut = (recOut)sprintf("%s", \
									substr(spaces, 1, formatLength(theFieldFormat[i])))
					}
				}
			}

			# update the previous value
			if (length(currVal[i]) != 0) { prevVal[i] = currVal[i] }
		}

		# update counts
		if (noInitialValue == 1) { initialRecsSkipped++ }

		# if it is not to be skipped, output the record
		if (recIsSkipped != 1) {
			numRecordsOutput++
			printf("%s", recOut)

			# output an EOL
			if (formulasAreUsed == 1) { print " \"?\"" }
			else { print "" }
		}
	}
	END {
		# output any non-zero counts information
		if (recsMissingRequiredFieldSkipped > 0) {
			system("echo >&2 ; echo NOTE: ignored "sprintf("%\047d", recsMissingRequiredFieldSkipped)" CSV record\\(s\\) that were missing values for one or more required field\\(s\\) >&2")
		}

		if (initialRecsSkipped > 0) {
			system("echo >&2 ; echo NOTE: when filling empty fields, ignored "sprintf("%\047d", initialRecsSkipped)" initial CSV record\\(s\\) with empty field\\(s\\) and no previous value >&2")
		}

		if (formulaAppliedToNonNumeric > 0) {
			system("echo >&2 ; echo NOTE: one or more formulas were not applied because the field is non-numeric >&2")
		}

		# output the number of data records in result file
		system("echo >&2 ; echo "sprintf("%\047d", numRecordsOutput)" data records were selected \\(plus the header\\) >&2")
	}' | \
# if using formulas, process using bc utility -- and/or do "clean-up" filtering
eval "$fCMD" >> "$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"

if test "$SHOW_SELECTED_FIELDS" = '1'
then
	echo "\nNOTE that the results file"
	echo "$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"
	echo "contains the selected records/fields with any requested sorting applied"
	echo "prior to the major field processing (req'd fields, field-filling, formulas)"
	/bin/rm -f "$WORK_FILE_PATHNAME"
	exit
else
	echo "\nGenerated $DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"
fi

# stats stuff ------------------------------------------------------------------
# if required, generate stats info
if test "$GENERATE_STATS" = '1'
then
	# setup the output file for the stats
	if test "$SEPARATE_STATS_FILE" = '1'
	then
		echo "\nGenerating statistics ..."
		STATS_FILE="$DIR/${FILE_NAME}${RESULTS_FILE_NAME_SUFFIX}-stats.$FILE_SUFFIX"
		/bin/echo -n > "$STATS_FILE"
	else
		echo "\nGenerating statistics (appended to results file) ..."
		STATS_FILE="$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX"
	fi

	# remove the whitespace at the start/end of entry-lines and adjacent to the
	# comma field-separators
	/usr/bin/sed -e 's/^[ ]*//'   \
					 -e 's/[ ]*$//'   \
					 -e 's/[ ]*,/,/g' \
					 -e 's/,[ ]*/,/g' \
						"$DIR/${FILE_NAME}$RESULTS_FILE_NAME_SUFFIX.$FILE_SUFFIX" | \
	# compute/output the statistics
	/usr/bin/awk -F ',' '
		(NR == 1) {
			# load some shell variables into awk
			separateStats = "'"$SEPARATE_STATS_FILE"'" + 0
			pageWidth = "'"$PAGE_WIDTH"'" + 0
			fieldWidth = "'"$FIELD_WIDTH"'" + 0

			# initialize a "number of (valid/processed) records" counter
			numRecords = 0

			# create an array of field names (assumes 1st record is the header)
			numFields = split($0, fieldName, ",")

			# initialize various arrays
			for (i = 1; i <= numFields; i++) {
				# create an "is initialized" array with 1 element for each field --
				# since empty/null fields are allowed, we need to know when each
				# field value has first appeared, so we can initialize the min/max
				didInit[i] = 0

				# create an initialized "maximum field length" array using the
				# specified field width
				# (i.e., ensure that the _minimum_ field length is "fieldWidth")
				maxLen[i] = fieldWidth + 0  # ensure it is a number

				# create initialized arrays of "previous value", "total of all
				# values", "number of values" and "total number of value changes"
				prevVal[i] = 0; sumVals[i] = 0; numVals[i] = 0; numChgs[i] = 0
			}
		}
		(NR > 1) {
			numRecords++  # keep a total of the number of records processed

			# determine the min/max values, the total of all values, the number of
			# value changes and the longest/required field lengths
			for (i = 1; i <= numFields; i++) {
				# get the field length
				fieldLen = length($i)

				# if applicable, update the "maximum field length" array
				if (fieldLen > maxLen[i]) { maxLen[i] = fieldLen }

				# update the minimum-value, maximum-value, total of all values,
				# number of values and number of changed values, as required
				if (fieldLen != 0) {  # only process non-empty field entries
					if (didInit[i] == 0) {
						# the field is not yet initialized, set the initial values
						min[i] = $i; max[i] = $i; sumVals[i] = $i
						prevVal[i] = $i; numVals[i] = 1; didInit[i] = 1
					}
					else {
						# field is initialized, update min/max, totals and changes
						# values, as applicable
						if ($i < min[i]) { min[i] = $i }
						if ($i > max[i]) { max[i] = $i }
						if ($i != prevVal[i]) { numChgs[i]++; prevVal[i] = $i }
						sumVals[i] = sumVals[i] + $i
						numVals[i]++
					}
				}
			}

			# if an average value and/or a number of changes value is larger
			# than the current field width, update the max field length array
			for (i = 1; i <= numFields; i++) {
				# get the field length
				chgsLen = length(numChgs[i])

				# value: >=1000 ->integer, >=1 ->2 decimal pts, <1 ->3 decimal pts
				if (numVals[i] != 0) { avgVal = sumVals[i] / numVals[i] }
				else { avgVal = 0 }

				if (avgVal >= 1000) { avgsLen = length(sprintf("%d", avgVal)) }
				else if (avgVal >= 1) {
					avgsLen = length(sprintf("%0.2f", avgVal))
				}
				else { avgsLen = length(sprintf("%0.3f", avgVal)) }

				# if applicable, update the "maximum field length" array
				if (avgsLen > maxLen[i]) { maxLen[i] = avgsLen }
				if (chgsLen > maxLen[i]) { maxLen[i] = chgsLen }
			}
		}
		END {
			# show the number of records processed
			if (separateStats != 1) { printf("\n") }
			printf("Records processed = %\047d (excluding header)\n", numRecords)

			# the length of the labels "field #: ", "  field: ", etc.
			labelWidth = 9

			# set a space-padding string used to right-justify printed fields
			spcStr = "                                                  "
			spcStr = spcStr""spcStr""spcStr

			# set a dash string to be used for header dividers
			dshStr = "--------------------------------------------------"
			dshStr = dshStr""dshStr""dshStr""dshStr""dshStr

			# initialize the number of fields that have been "done"/processed
			fieldsDone = 0

			while (fieldsDone < numFields) {
				# determine the number of fields that fit within the page width
				# (minus the length of the field/minimum/maximum labels)
				fieldsInPageWidth = 0
				remainingWidth = pageWidth - labelWidth + 2

				for (i = (1 + fieldsDone); i <= numFields; i++) {
					remainingWidth = remainingWidth - (maxLen[i] + 2)

					if (remainingWidth >= 0) { fieldsInPageWidth++ }
					else { break }
				}

				# allow last field that is longer than the page width
				if (fieldsInPageWidth == 0) { fieldsInPageWidth = 1 }

				# determine the last field number in this group
				endFieldNum = fieldsDone + fieldsInPageWidth
				if (endFieldNum > numFields) { endFieldNum = numFields }

				# print info in groups that fit within the page width
				startOfGroup = 1

				# print a group of field names
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					# determine the field name length to use for printing
					fieldLen = length(fieldName[i])

					# determine the field name to be used for printing
					if (fieldLen > (maxLen[i] + 1)) {  # allow 1 space separation
						# field name needs to be truncated
						fName = substr(fieldName[i], 1, (maxLen[i] + 1))
					}
					else {  # field name needs to be padded
						# determine the right-justifying pad string and add to name
						fName = \
							substr(spcStr, 1, ((maxLen[i] + 1) - fieldLen))fieldName[i]
					}

					# print a group of field headings to help associate the
					# subsequent values with their meanings
					if (startOfGroup == 1) {
						if (i > 1) { print "" }
						printf("\n  field|")
						printf("%s", fName)
						startOfGroup = 0
					}
					else { printf("|%s", fName) }
				}

				startOfGroup = 1

				# print a group of field numbers
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					fNum = sprintf("%s", i)

					# determine the field number length to use for printing
					fieldLen = length(fNum)

					# if the field number is too long, it is "blanked" because a
					# truncated field number would be wrong (although there are a
					# minimum of 2 spaces of separation between min/max values, we
					# allow a minimum of 1 space separation for the field numbers
					if (fieldLen > (maxLen[i] + 1)) { fNum = " " }

					# pad the field number with leading spaces, as required
					fNum = substr(spcStr, 1, ((maxLen[i] + 1) - fieldLen))fNum

					# print a field number for this group
					# min/max values with their meanings
					if (startOfGroup == 1) {
						printf("\nfield #|")
						printf("%s", fNum)
						startOfGroup = 0
					}
					else { printf("|%s", fNum) }
				}

				startOfGroup = 1

				# print a group of heading dividers
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					# determine the field header divider string
					divStr = substr(dshStr, 1, maxLen[i] + 1)

					if (startOfGroup == 1) {
						printf("\n-------|")
						printf("%s", divStr)
						startOfGroup = 0
					}
					else { printf("|%s", divStr) }
				}

				startOfGroup = 1

				# print a group of minimum values
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					if (didInit[i] == 0) { minStr = "n/a" }
					else { minStr = sprintf("%s", min[i]) }

					# determine the right-justifying pad string
					fieldLen = length(minStr)
					minStr = substr(spcStr, 1, (maxLen[i] - fieldLen))minStr

					if (startOfGroup == 1) {
						printf("\nminimum| ")
						printf("%s", minStr)
						startOfGroup = 0
					}
					else { printf("| %s", minStr) }
				}

				startOfGroup = 1

				# print a group of maximum values
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					if (didInit[i] == 0) { maxStr = "n/a" }
					else { maxStr = sprintf("%s", max[i]) }

					# determine the right-justifying pad string
					fieldLen = length(maxStr)
					maxStr = substr(spcStr, 1, (maxLen[i] - fieldLen))maxStr

					if (startOfGroup == 1) {
						printf("\nmaximum| ")
						printf("%s", maxStr)
						startOfGroup = 0
					}
					else { printf("| %s", maxStr) }
				}

				startOfGroup = 1

				# print a group of heading dividers
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					# determine the field header divider string
					divStr = substr(dshStr, 1, maxLen[i] + 1)

					if (startOfGroup == 1) {
						printf("\n-------|")
						printf("%s", divStr)
						startOfGroup = 0
					}
					else { printf("|%s", divStr) }
				}

				startOfGroup = 1

				# print a group of maximum-minimum/"range" values
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					if ((didInit[i] == 0) ||
						 (min[i] !~ /^-{0,1}[0-9]*\.{0,1}[0-9]+$/) ||
						 (max[i] !~ /^-{0,1}[0-9]*\.{0,1}[0-9]+$/)) {
						diffStr = "n/a"
					}
					else { diffStr = sprintf("%s", max[i] - min[i]) }

					# determine the right-justifying pad string
					fieldLen = length(diffStr)
					diffStr = substr(spcStr, 1, (maxLen[i] - fieldLen))diffStr

					if (startOfGroup == 1) {
						printf("\nmax-min| ")
						printf("%s", diffStr)
						startOfGroup = 0
					}
					else { printf("| %s", diffStr) }
				}

				startOfGroup = 1

				# print a group of average values
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					if ((didInit[i] == 0) || (numVals[i] == 0) ||
						 (min[i] !~ /^-{0,1}[0-9]*\.{0,1}[0-9]+$/) ||
						 (max[i] !~ /^-{0,1}[0-9]*\.{0,1}[0-9]+$/)) {
						avgStr = "n/a"
					}
					else {
						# if value: >=1000 use integer, >=10 use 2 decimal pts,
						#           <10 use 3 decimal pts
						avgVal = sumVals[i] / numVals[i]

						if (avgVal >= 1000) { avgStr = sprintf("%d", avgVal) }
						else if (avgVal >= 1) { avgStr = sprintf("%0.2f", avgVal) }
						else { avgStr = sprintf("%0.3f", avgVal) }
					}

					# determine the right-justifying pad string
					fieldLen = length(avgStr)
					avgStr = substr(spcStr, 1, (maxLen[i] - fieldLen))avgStr

					if (startOfGroup == 1) {
						printf("\naverage| ")
						printf("%s", avgStr)
						startOfGroup = 0
					}
					else { printf("| %s", avgStr) }
				}

				startOfGroup = 1

				# print a group of "number of changes" values
				for (i = (1 + fieldsDone); i <= endFieldNum; i++) {
					if (didInit[i] == 0) {
						chgsStr = "n/a"
					}
					else { chgsStr = sprintf("%s", numChgs[i]) }

					# determine the right-justifying pad string
					fieldLen = length(chgsStr)
					chgsStr = substr(spcStr, 1, (maxLen[i] - fieldLen))chgsStr

					if (startOfGroup == 1) {
						printf("\nchanges| ")
						printf("%s", chgsStr)
						startOfGroup = 0
					}
					else { printf("| %s", chgsStr) }
				}

				fieldsDone = fieldsDone + fieldsInPageWidth
			}

			# print the complete list of fields (since they can be truncated when
			# shown in the field header(s)
			printf("\n\nfields\n")

			# for each of the applicable fields, output the field
			for (i = 1; i <= numFields; i++) {
				printf("%3d: %s\n", i, fieldName[i])
			}
		}' >> "$STATS_FILE"

	if test "$SEPARATE_STATS_FILE" = '1'
	then
		echo "\nGenerated $STATS_FILE"
	fi
fi

# cleanup ----------------------------------------------------------------------
/bin/rm -f "$WORK_FILE_PATHNAME"

# reset the IFS (in case this script gets "inlined" into another script)
IFS=$IFS_SAVE