omatrix.py

SIMO optimization data matrix:

>>> import numpy
>>> import datetime
>>> import os
>>> from minimock import Mock
>>> epsilon = 0.00001

def calculate_NPV(value, discountrate, presentdate, eventdate):

Calculate the discounted NPV (Net Present Value) for a single value or vector:

>>> from simo.optimization.tools.omatrix import calculate_NPV
>>> npv = calculate_NPV(100.0, 5.0,
...                     datetime.date(2000,1,1), datetime.date(2010,1,1))
>>> abs(npv - 61.391325) < epsilon
True
>>> values = numpy.array([100, 100, 100], dtype=float)
>>> npv = calculate_NPV(values, 5.0,
...                     datetime.date(2000,1,1), datetime.date(2010,1,1))
>>> abs(npv - 61.391325) < epsilon
array([ True,  True,  True], dtype=bool)

class OMatrix(object):

Class for in-memory storage of SIMO optimization data structures. Get the wanted data from input database, stores it into multidimensional numpy arrays for fast access during optimization process.

Data is stored for each unique subobjective and constraint operand into dictionaries where keys are the unique operand indices and values are threedimensional Numpy double arrays.

The data arrays have dimensions: (object, branch, timestep)

def __init__(self):

>>> taskdef1 = Mock('TaskDef')
>>> taskdef1.main_level = u'comp_unit'
>>> taskdef1.num_of_subobjs = 1
>>> taskdef1.num_of_consts = 1
>>> taskdef1.unique_operands = {('variable',
...                              (u'comp_unit', 'PV'),
...                              None,
...                              False): 0,
...                             ('operation',
...                              'cash_flow',
...                              None,
...                              False): 1}
>>> taskdef1.operand_index = {0: 0, 1: 1}
>>> taskdef1.one_aggr_level = {0: True, 1:False}
>>> taskdef1.date_ranges = {0: (datetime.date(1998,1,1),None),
...                         1: (datetime.date(1998,1,1),None)}
>>> taskdef1.discount_rate = 10.0
>>> taskdef1.init_date = datetime.date(1998,1,1)
>>> taskdef1.subobjectives = [Mock('Subobjective')]
>>> taskdef1.subobjectives[0].expr = [('variable',
...                                   (u'comp_unit', 'PV'),
...                                   (datetime.date(1998,1,1),None),
...                                   None,
...                                   False, (0, 0))]
>>> taskdef1.constraints = [[('operation',
...                           'cash_flow',
...                           (datetime.date(1998,1,1),None),
...                           None,
...                           False, (1, 1))]]
>>> add_error = Mock('add_error')

Initialize OMatrix:

>>> from simo.optimization.tools.omatrix import OMatrix
>>> run_id = 'test'
>>> keep_data = False
>>> use_existing= False
>>> om = OMatrix(taskdef1, run_id, use_existing, keep_data, add_error)

def analyze_data(self, db, add_info):

Parameters

db -- input database handle
add_info -- logger function object

Go through subobjectives, constraints and input database and analyze some data properties

>>> db = Mock('DB')
>>> db.get_ids.mock_returns = [u'001', u'002', u'003']
>>> db.get_max_iter.mock_returns = 1
>>> db.get_max_branch.mock_returns = 1
>>> db.get_dates.mock_returns = [datetime.date(1998,12,31),
...                              datetime.date(1999,12,31)]
>>> db.get_data_from_level.mock_returns = [
...     ['001', 0, datetime.date(1998,12,31), 1000.0],
...     ['001', 1, datetime.date(1998,12,31), 1010.0],
...     ['001', 0, datetime.date(1999,12,31), 1001.0],
...     ['001', 1, datetime.date(1999,12,31), 1011.0],
...     ['002', 0, datetime.date(1998,12,31), 2000.0]]
>>> add_info = Mock('logger.add_message')

>>> om.analyze_data(db, add_info) # doctest: +ELLIPSIS
Called logger.add_message('  analysing input database')
Called DB.get_ids('data', u'comp_unit')
Called DB.get_max_iter('data', u'comp_unit')
Called DB.get_max_branch('data', u'comp_unit')
Called logger.add_message('  number of simulated iterations: 2')
True

def construct_data(self, iteration):

Parameters:

iteration -- current simulated iteration indice as int

Construct optimization data matrix and fill it with data from input databases

>>> evaluator = Mock('Evaluator')
>>> evaluator.evaluate.mock_returns = True
>>> om._evaluator = evaluator  # substitute the real evaluator with a mock
>>> _add_info = Mock('_add_info')
>>> om.construct_data(0)  # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Called logger.add_message('Constructing optimization data set for iteration 0')
Called logger.add_message('  creating optimisation tables')
Called DB.get_dates('op_res', None, (datetime.date(1998, 1, 1), None))
Called DB.get_dates('data', u'comp_unit', (datetime.date(1998, 1, 1), None))
Called DB.get_dates('data', u'comp_unit', (datetime.date(1998, 1, 1), None))
Called logger.add_message(
    '  number of unique subobjective/constraint operands: 2')
Called logger.add_message('  total size for optimization data arrays: 192.00b')
Called logger.add_message('  filling data')
Called DB.db.index_exists(u'comp_unit_iter_date_idx')
Called logger.add_message('  preparing db for optimization data collection...')
Called DB.db.execute(
    u'CREATE INDEX comp_unit_iter_date_idx ON "comp_unit" (iteration, date)')
Called DB.db.index_exists('op_link_iter_date_idx')
Called DB.db.execute(
    'CREATE INDEX op_link_iter_date_idx ON op_link (iteration, date)')
Called DB.db.execute('ANALYZE')
Called DB.get_data_from_level(
    'op_res',
    ['id', 'branch', 'date', 'cash_flow'],
    0,
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    required=['cash_flow'])
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'date', 'PV'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None))
Called Evaluator.evaluate(
    array([0, 1, 2]),
    0,
    [('variable', (u'comp_unit', 'PV'), (datetime.date(1998, 1, 1), None), None, False, (0, 0))],
    {0: (0, 1), 1: (0, 1)},
    {0: array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32), 1: array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32)},
    3,
    False)
Called Evaluator.evaluate(
    array([0, 1, 2]),
    1,
    [('variable', (u'comp_unit', 'PV'), (datetime.date(1998, 1, 1), None), None, False, (0, 0))],
    {0: (0, 1), 1: (0, 1)},
    {0: array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32), 1: array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32)},
    3,
    False)
Called logger.add_message('  optimization data arrays filled in 00:00:0.0...')
Called logger.add_message('  data set ready for optimization')
True



>>> om.data_level
u'comp_unit'
>>> om._earliest_date
datetime.date(1998, 1, 1)
>>> om._latest_date
>>> om.max_iterations
2
>>> om.num_of_units
3
>>> om._max_branches
2
>>> numpy.all(numpy.array(om.num_of_branches) == numpy.array([2, 1, 1]))
True
>>> om._date_index[0][datetime.date(1998,12,31)]
0
>>> om._date_index[0][datetime.date(1999,12,31)]
1
>>> om._date_index[1][datetime.date(1998,12,31)]
0
>>> om._date_index[1][datetime.date(1999,12,31)]
1
>>> om.unit_index[u'001']
0
>>> om.unit_index[u'002']
1
>>> om.unit_index[u'003']
2
>>> om._unit_range
array([0, 1, 2])
>>> data = om.data['operands']
>>> data[0]
array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32)
>>> data[1]
array([[[ 1000.,  1001.],
        [ 1010.,  1011.]],
<BLANKLINE>
       [[ 2000.,     0.],
        [    0.,     0.]],
<BLANKLINE>
       [[    0.,     0.],
        [    0.,     0.]]], dtype=float32)
>>> data[0].shape
(3, 2, 2)
>>> data[1].shape
(3, 2, 2)

Additional tests for initialization

Initialize to keep the optimization data:

>>> run_id = 'test2'
>>> keep_data = True
>>> use_existing = False
>>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data, add_error)
>>> om2._evaluator = evaluator
>>> om2.analyze_data(db, add_info) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called ...
True
>>> om2.construct_data(0) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called ...
True
>>> os.path.exists('test2.dat')
False
>>> om2.close()
Called logger.add_message('Writing optimization data structure to file...')
>>> del om2
>>> os.path.exists('test2.dat')
True

Initialize to reuse existing optimization data file:

>>> keep_data = False
>>> use_existing = True

NB! The commented out section should work but doesnt’t... test2.h5 exists up to call to om2.construct_data(0) when it just disappears...

# >>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data, add_error) # >>> om2._evaluator = evaluator # >>> om2.analyze_data(db, add_info) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE # Called ... # True # >>> om2.construct_data(0) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE # Called ... # True # >>> os.path.exists(‘test2.data’) # True # >>> om2.close() # >>> os.path.exists(‘test2.data’) # False

>>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data, add_error)
>>> om2._evaluator = evaluator  # substitute the real evaluator with a mock
>>> om2.analyze_data(db, add_info) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called ...
True
>>> os.remove(om2._data_file)
>>> om2.construct_data(0) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called ...
...'Reuse optimization data file set, but test2.dat does not exist'...
False

def _create_tables(self):

Create optimization data tables, using Numpy arrays for the unique operand arrays.

The data is stored in a dictionary where keys are unique operand indices and values are unique operand data arrays.

Unique operand (subobjective or constraint) array shape: (units, branches, timesteps)

def _process_exprs(self, exprs, db, exprtype, subobj=True):

Go through the expressions and try to deduce data level and earliest and latest dates

def _fill_tables(self, it):

Fill subobjective and constraint arrays with values from input databases.

Parameters

it -- current simulated iteration indice, int

def _collect_values(self, it, operand, array, ui, forced_dates=None):

Get values for a single expression operand from either data or operation result database.

Parameters

it -- current simulated iteration indice as int
operand -- subobjective or constraint operand
array -- target Numpy array
ui -- unique subobjective/constraint operand indice as int
forced_dates -- forced date info

Collect values for a simple expression with only ‘variable’ type operands

>>> om._db = db
>>> array1 = numpy.zeros([2,3,2], dtype=float)
>>> array2 = numpy.zeros([2,3,2], dtype=float)

>>> operand1 = ('variable', (u'comp_unit', 'PV'), None, False)
>>> operand2 = ('variable', (u'comp_unit', 'AREA'), None, True)

>>> om._collect_values(0, operand1, array1, 0)
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'date', 'PV'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None))
True

>>> om._collect_values(0, operand2, array2, 0)
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'date', 'AREA'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None))
True

>>> array1[:,:,0]
array([[ 1000.,  1010.,     0.],
       [ 2000.,     0.,     0.]])

>>> array2[:,:,0]
array([[ 1000.,  1010.,     0.],
       [ 2000.,     0.,     0.]])

Collect values for an expression with ‘operation’ type operands

>>> array1 = numpy.zeros([2,3,2], dtype=float)
>>> array2 = numpy.zeros([2,3,2], dtype=float)
>>> operand1 = ('operation', u'cash_flow', None, True)
>>> operand2 = ('variable', (u'comp_unit', 'AREA'), None, False)

>>> om._collect_values(0, operand1, array1, 0)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called DB.get_data_from_level(
    'op_res',
    ['id', 'branch', 'date', u'cash_flow'],
    0,
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    required=[u'cash_flow'])
True

>>> om._collect_values(0, operand2, array2, 0)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'date', 'AREA'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None))
True

Collect values for an expression with conditions in ‘variable’ type operands

>>> array = numpy.zeros([2,3,2], dtype=float)
>>> dt = (datetime.date(1998,12,31), None)
>>> cond = [('variable',u'SP'), ('value',1000), ('gt',(lambda a,b: a>b), 'gt')]
>>> operand = ('variable', (u'comp_unit', 'PV'), cond, False)
>>> om._collect_values(0, operand, array, 0)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'date', 'PV'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0, 'and'), (u'SP', 'gt', 1000)],
    dates=(datetime.date(1998, 1, 1), None))
True

Collect values for an expression with conditions in ‘operation’ type operands

>>> operand = ('operation', u'cash_flow',
...            [('operation', u'operation_name'),
...             ('value', u'thinning'),
...              ('eq', (lambda a,b: a == b), 'eq')], True)

>>> db = Mock('DB')
>>> db.get_data_from_level.mock_returns = \
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['002', 0, datetime.date(1999,12,31), 3000.0],
...      ['002', 1, datetime.date(1999,12,31), 4000.0]]
>>> om._db = db
>>> om._evaluator = Mock('Evaluator')
>>> om._evaluator.evaluate.mock_returns_iter = iter([True, False, True, True])
>>> array[...] = 0.0
>>> om._collect_values(0, operand, array, 0)  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
Called DB.get_data_from_level(
    'op_res',
    ['id', 'branch', 'date', u'cash_flow'],
    0,
    constraints=[('iteration', 'eq', 0, 'and'), (u'operation_name', 'eq', u'thinning')],
    dates=(datetime.date(1998, 1, 1), None),
    required=[u'cash_flow'])
True

>>> array[...]
array([[[ 1000.        ,     0.        ],
        [    0.        ,     0.        ],
        [    0.        ,     0.        ]],
<BLANKLINE>
       [[    0.        ,  2727.27272727],
        [    0.        ,  3636.36363636],
        [    0.        ,     0.        ]]])

def _get_values_from_db(self, it, operand, dates, norm, condition=None):

Get value(s) for a single operand from data or operation result database

Parameters:

it – current simulated iteration indice as int operand – current operand definition dates – start and end dates norm – boolean indicating whether the operand is normal condition – operand condition/constraint

def _eval_constraint(self, it, dates, array, operand, ui, norm):

Evaluate operand condition and store only those objects for which the condition evaluated True.

Parameters:

it – current simulated iteration indice as int dates – start and end dates array – target Numpy array operand – operand structure ui – unique subobjective/constraint location indice norm – boolean indicating whether the operand is normal

def _prepare_db(self):

Makes sure the proper indices exist and that their content is up to date.

def _set_value(self, array, uid, br, ui, date, value):

Store a single value into a correct location in a matrix and update branch number container.

Parameters:

value – Numpy array uid – object id, str br – branch, int ui – unique subobjective/constraint indice, int date – datetime object value – value to store, float

def solution_utility(self, solution):

Check objective function utility value in given solution

def _value2utility(self, iso, value, maxvalue):

Transform real values to utilities, a Cython extension function

def solution_feasibility(self, solution):

Check given solution feasibility. For the initial seek for feasible solution goes through all the constraints in order to be able to report back to the user the constraint feasible/not feasible ratio in case of not finding any feasible solutions.

def compare_utilities(self, best, candidate):

Compare utility values of two solutions: is candidate better than the current best solution?

>>> om.task_def.objective_func.target = 'max'
>>> om.compare_utilities(0.0, 1.0)
True
>>> om.compare_utilities(2.0, 1.0)
False

>>> om.task_def.objective_func.target = 'min'
>>> om.compare_utilities(0.0, 1.0)
False
>>> om.compare_utilities(2.0, 1.0)
True

def get_random_solution(self):

Get a random solution without checkin it’s feasibility

>>> om.get_random_solution().shape
(3,)

def get_status(self, solution):

Get a data structure with string representations and current values of all subobjectives and constraints.

Parameters:

iteration – current iteration solution – current solution