.. _aggr-py: 

#######
aggr.py
#######

*********************************
class OutputAggr(Output):
*********************************

Module for data output in hierarchical "data in columns" format. (writing tables to files)

def __init__(self, datadb, data_type, id_list, main_level, result_type, output_formats, output_filenames, output_constraint=None, default_decimal_places=1, archiving=False, result_padding=True, aggregation_def=None, expression_def=None, opres_vars=None):
==============================================================================================================================================================================================================================================================

Creates the output class. fully inherited, executes self.run() ::

    >>> from simo.output.test.init_objects import *
    >>> from collections import defaultdict
    >>> import datetime
    >>> import numpy
    >>> idata = InitData()
    >>> idata.init()
    >>> testdb = idata.testdb
    >>> const_obj = idata.const_obj
    >>> aggr_obj = idata.aggr_obj
    >>> expr_obj = idata.expr_obj
    >>> from simo.output.aggr import OutputAggr
    >>> out = OutputAggr(testdb, 'result', ['stand1'], 'stratum', 
    ...               'optimized',
    ...               'output/test/aggregation.txt', TestLogger(),
    ...               const_obj, 1, False, True, aggr_obj, expr_obj,
    ...               ['cash_flow', 'Income', 'Scrapwood', 'Volume', 
    ...               'BIOMASS_branches', 'BIOMASS_stumps'])
    >>> try: # doctest: +ELLIPSIS
    ...     file = open('output/test/aggregation.txt', 'r')
    ...     for line in file:
    ...         print line.rstrip('\n')
    ... finally:
    ...     file.close() #doctest: +ELLIPSIS
    Results for expression (5 keys): sum[-1:-1](stratum:HgM[stratum:SP eq 2 and stratum:DgM gt 19])
     stratum:SP; stratum:DgM; 08/02/09-08/02/09
            1.0;        20.0;               0.0
            2.0;         0.0;               0.0
            2.0;        18.0;               0.0
            2.0;        30.0;              62.0
            3.0;        27.0;               0.0
    <BLANKLINE>
    Proportions for expression (5 keys): sum[0:99](stratum:SP*stratum:HgM)
     stratum:HgM; stratum:DgM; 01/01/09-08/02/09
             0.0;         0.0;               0.0
            18.5;        18.0;               0.2
            19.0;        20.0;               0.1
            26.0;        27.0;               0.4
            31.0;        30.0;               0.3
    <BLANKLINE>
    Results for expression (1 keys): sum[0:99](operation:Volume)
     01/01/09-08/02/09
                 123.2
    <BLANKLINE>
    Results for expression (1 keys): sum[0:99](operation:cash_flow)
     01/01/09-08/02/09
                3000.0
    <BLANKLINE>
    Results for expression (2 keys): sum[0:99](operation:cash_flow)
     comp_unit:AREA; 01/01/09-08/02/09
                1.0;            3000.0
                2.0;               0.0
    <BLANKLINE>
    Results for expression (2 keys): sum[0:99](operation:cash_flow)
     operation:op_name; comp_unit:SC; comp_unit:DEV_CLASS; 01/01/09-08/02/09
              clearcut;          3.0;                 4.0;            3456...
              planting;          3.0;                 4.0;            -456...
    <BLANKLINE>


def run(self):
==============

Uses data from init to run the class-specific output (self.aggregation)

def _run_aggr(self, aggr_dict):
===============================

Executes the given aggregation dictionary for all reporting periods and return the results in a list of lists of strings.

def _execute_var(self, discount_rate, time_unit, time_length, var):
===================================================================

Executes the given aggregation set with the given values and returns results as a list of strings.

def _parse_experssion(self, expression, header, init_date):
===========================================================

Will parse a string expression into a list of tuples. Returns None if there are errors. ::

    >>> out._parse_expression('sum[3:5](comp_unit:AREA)', 
    ...                       'sum[3:5](comp_unit:AREA)', 
    ...                       datetime.date(2009, 1, 1)) 
    ... #doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
    [('data', ('comp_unit', u'AREA'), 
     (datetime.date(2011, 1, 1), datetime.date(2013, 12, 31)), None, False), 
     ('aggr', <function sum at ...>)]

def _get_classifiers(self, keys):
=================================

Build and return classifiers for the given grouping ::

    >>> out._get_classifiers(['comp_unit:AREA', 'operation:Volume'])
    [(1.0, 123.2)]

def _get_values(self, expr, expression, date_ranges, grouping, errors, date_dict):
==================================================================================

Get the values from the database into the date_dict ::
    
    >>> expr = [('op', u'Volume', (datetime.date(2009, 2, 7), 
    ...          datetime.date(2108, 2, 6)), None, False), ('aggr', numpy.sum)]
    >>> expression = 'Test expression string'
    >>> date_ranges = [(datetime.date(2009, 2, 7), datetime.date(2009, 2, 8), 
    ...                 'year', True)]
    >>> grouping = None
    >>> errors = defaultdict(set)
    >>> date_dict = {}
    >>> date_dict, classifiers, errors = out._get_values(expr, expression,
    ...                                                  date_ranges, 
    ...                                                  grouping, errors, 
    ...                                                  date_dict, 
    ...                                                datetime.date(2009,2,7),
    ...                                                3.0)
    >>> date_dict, classifiers, errors #doctest: +NORMALIZE_WHITESPACE
    ({(datetime.date(2009, 2, 7), datetime.date(2009, 2, 8), 'year', True): 
        defaultdict(<type 'list'>, 
     {'': [array([ 123.2])]})}, [''], defaultdict(<type 'set'>, {}))

def _get_results(self, date_dict, grouping, classifiers, expr, proportion):
===========================================================================

Evaluate the expression and return a list of results and the header for the result grouping ::

    >>> proportion = False
    >>> grouping = {}
    >>> out._get_results(date_dict, grouping, classifiers, expr, proportion)
    ([['07/02/09-08/02/09'], ['123.2']], [])

def _check_date_range(self, range, val):
========================================

Check if the val is inside the given range ::

    >>> out._check_date_range(date_ranges[0], datetime.date(2009, 2, 6))
    False
    >>> out._check_date_range(date_ranges[0], datetime.date(2009, 2, 7))
    True
    >>> out._check_date_range(date_ranges[0], datetime.date(2009, 2, 8))
    True
    >>> out._check_date_range((datetime.date(2009, 2, 7), 
    ...                        datetime.date(2009, 2, 8), 'year', False), 
    ...                        datetime.date(2009, 2, 8))
    False

def _get_date_ranges(self, init_date, all_dates, expr, time_unit, time_length):
===============================================================================

This gets the date ranges to be used for the data ::

    >>> init_date = datetime.date(2009, 2, 7)
    >>> all_dates = [datetime.date(2009, 2, 7), datetime.date(2009, 3, 1)]
    >>> out._get_date_ranges(init_date, all_dates, expr, 'day', 5)
    ... #doctest: +NORMALIZE_WHITESPACE
    [(datetime.date(2009, 2, 7), datetime.date(2009, 2, 12), 'day', False), 
     (datetime.date(2009, 2, 12), datetime.date(2009, 2, 17), 'day', False), 
     (datetime.date(2009, 2, 17), datetime.date(2009, 2, 22), 'day', False), 
     (datetime.date(2009, 2, 22), datetime.date(2009, 2, 27), 'day', False), 
     (datetime.date(2009, 2, 27), datetime.date(2009, 3, 1), 'day', True)]

def _build_ranges(self, dates, time_unit, time_length, first=None, last=None):
==============================================================================

Returns a list of tuples of (start_date, end_date, time_unit, is_last_range) ::

    >>> out._build_ranges(all_dates, 'day', 5, first=datetime.date(2009,2,20),
    ...                   last=datetime.date(2009,2,28)) 
    ... #doctest: +NORMALIZE_WHITESPACE
    [(datetime.date(2009, 2, 20), datetime.date(2009, 2, 25), 'day', False), 
     (datetime.date(2009, 2, 25), datetime.date(2009, 2, 28), 'day', True)]

def _get_string_range(self, rng):
=================================

Turns the given range tuple into a string representation ::

    >>> out._get_string_range((datetime.date(2009, 2, 20), 
    ...                        datetime.date(2009, 3, 25), 'day', False))
    '20/02/09-24/03/09'
    >>> out._get_string_range((datetime.date(2009, 2, 20), 
    ...                        datetime.date(2009, 3, 25), 'month', True))
    '20/02/09-25/03/09'
    >>> out._get_string_range((datetime.date(2009, 2, 20), 
    ...                        datetime.date(2009, 3, 25), 'year', True))
    '20/02/09-25/03/09'

def _unpack_weighted_average(self, data):
=========================================

Unpacks a weighted average into a formula the expression parser can handle. ::

    >>> out._unpack_weighted_average('sum[1:5](one:thing)')
    'sum[1:5](one:thing)'
    >>> out._unpack_weighted_average('wavg[1:5](one:thing, another:thing)')
    'sum[1:5](one:thing*another:thing)/sum[1:5](another:thing)'

def _get_values_from_db(self, operand, dates, date_ranges, norm, 
                        opresconst=None, grouping=None):
================================================================

Get value(s) for a single operand from data or operation result database. Valid grouping pairs (in either way): level-level, operation-"comp_unit", level-child, level-parent ::

    >>> r = out._get_values_from_db(('data', ('stratum', u'SP'), 
    ...                         (datetime.date(2009, 2, 7), 
    ...                          datetime.date(2108, 2, 6)), None, False),
    ...                         (datetime.date(2009, 2, 7), 
    ...                          datetime.date(2108, 2, 6)), 
    ...                          [(datetime.date(2009, 1, 1), 
    ...                            datetime.date(2009, 2, 25), 'day', False), 
    ...                           (datetime.date(2009, 2, 25), 
    ...                            datetime.date(2009, 2, 28), 'day', True)],
    ...                          True, None,
    ...                          {'stratum': ['HgM', 'DgM']})
    >>> r[0].sort()
    >>> r #doctest: +NORMALIZE_WHITESPACE
    ([[1.0, datetime.date(2009, 2, 7), 19.0, 20.0], 
      [1.0, datetime.date(2009, 2, 7), 19.0, 20.0],
      [2.0, datetime.date(2009, 2, 7), 18.5, 18.0], 
      [2.0, datetime.date(2009, 2, 7), 18.5, 18.0], 
      [2.0, datetime.date(2009, 2, 8), 0.0, 0.0], 
      [2.0, datetime.date(2009, 2, 8), 0.0, 0.0], 
      [2.0, datetime.date(2009, 2, 8), 31.0, 30.0], 
      [2.0, datetime.date(2009, 2, 8), 31.0, 30.0], 
      [3.0, datetime.date(2009, 2, 8), 26.0, 27.0], 
      [3.0, datetime.date(2009, 2, 8), 26.0, 27.0]], 
      [datetime.date(2009, 2, 8), datetime.date(2009, 2, 7)], 
      ['stratum:HgM', 'stratum:DgM'])
    >>> out._get_values_from_db(('op', u'cash_flow', 
    ...                         (datetime.date(2009, 2, 7), 
    ...                          datetime.date(2108, 2, 6)), None, False),
    ...                         (datetime.date(2009, 2, 7), 
    ...                          datetime.date(2108, 2, 6)), 
    ...                         [(datetime.date(2009, 1, 1), 
    ...                           datetime.date(2009, 2, 25), 'day', False), 
    ...                          (datetime.date(2009, 2, 25), 
    ...                           datetime.date(2009, 2, 28), 'day', True)],
    ...                          True, None,
    ...                          {'comp_unit': ['AREA']})
    ... #doctest: +NORMALIZE_WHITESPACE
    ([[3000.0, datetime.date(2009, 2, 8), 1.0]], 
      [datetime.date(2009, 2, 8)], ['comp_unit:AREA'])

def _calculate_NPV(self, value, discountrate, presentdate, eventdate):
======================================================================

Calculate the discounted NPV (Net Present Value) for a single value ::
    
    >>> out._calculate_NPV(10.5, 5.4, datetime.date(2009,1,1), 
    ...                    datetime.date(2011,5,4))
    ... #doctest: +ELLIPSIS
    9.45165971144533...
    
def aggregation(self, file):
============================

This method will get the data requested from the databases and writes them into the given file. ::

    >>> out.aggregation('output/test/aggregation.txt')
    >>> try:
    ...     file = open('output/test/aggregation.txt', 'r')
    ...     for line in file:
    ...         print line.rstrip('\n')
    ... finally:
    ...     file.close() #doctest: +ELLIPSIS
    Results for expression (5 keys): sum[-1:-1](stratum:HgM[stratum:SP eq 2 and stratum:DgM gt 19])
     stratum:SP; stratum:DgM; 08/02/09-08/02/09
            1.0;        20.0;               0.0
            2.0;         0.0;               0.0
            2.0;        18.0;               0.0
            2.0;        30.0;              62.0
            3.0;        27.0;               0.0
    <BLANKLINE>
    Proportions for expression (5 keys): sum[0:99](stratum:SP*stratum:HgM)
     stratum:HgM; stratum:DgM; 01/01/09-08/02/09
             0.0;         0.0;               0.0
            18.5;        18.0;               0.2
            19.0;        20.0;               0.1
            26.0;        27.0;               0.4
            31.0;        30.0;               0.3
    <BLANKLINE>
    Results for expression (1 keys): sum[0:99](operation:Volume)
     01/01/09-08/02/09
                 123.2
    <BLANKLINE>
    Results for expression (1 keys): sum[0:99](operation:cash_flow)
     01/01/09-08/02/09
                3000.0
    <BLANKLINE>
    Results for expression (2 keys): sum[0:99](operation:cash_flow)
     comp_unit:AREA; 01/01/09-08/02/09
                1.0;            3000.0
                2.0;               0.0
    <BLANKLINE>
    Results for expression (2 keys): sum[0:99](operation:cash_flow)
     operation:op_name; comp_unit:SC; comp_unit:DEV_CLASS; 01/01/09-08/02/09
              clearcut;          3.0;                 4.0;            3456...
              planting;          3.0;                 4.0;            -456...
    <BLANKLINE>