.. _exprparser-py: ############# exprparser.py ############# SIMO optimization expression parser:: >>> epsilon = 0.00001 >>> from lxml import etree >>> import datetime ******************************* class ExpressionParser(object): ******************************* Optimization task expression parser Parameters: - validator: SIMO lexicon validator def __init__(self): =================== >>> from simo.builder.optimization.exprparser import ExpressionParser >>> parser = ExpressionParser() def parse(self, expr, initdate, step, aggregator_constraint=None): ================================================================== Parse SIMO optimization subobjective or constraint expression into a nested list and further into a Reverse Polish notation / Postfix stack. Parameters :: expr -- expression definition in a list structure initdate -- initial date as datetime object step -- time step unit, eg. 'year', 'month', 'day' aggregator_constraint -- valid aggregator function constraint, iterable eg. ['sum', 'min'] Expression is stored as a list of tuples. Each tuple is one of: operation or data variable definition, value definition, or function object. The first item of the tuple defines the type of the expression item. The expression is stored in postfix notation (reverse polish notation). Variable and operation tuple structure: - 0: 'data' or 'op' - 1: (level, variable) tuple if 'data', operation result variable if 'op' - 2: (start date, end date) tuple, None if date is -1 (stands for last possible date in the data) - 3: condition expression, given also in postfix notation, None if no condition - 4: discount, True if variable values should be discounted, False if not Value tuple structure: - 0: 'value' - 1: value as float Function tuple structure: - 0: 'eq', 'ari', or 'aggr' for equality, arithmetic, and aggregation functions respectively - 1: function object:: >>> initdate = datetime.date(2000, 1, 1) >>> step = 'year' Parse: sum[period](X):: >>> expr = u'sum[1:10](comp_unit:PV)' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'PV'), (datetime.date(2000, 1, 1), datetime.date(2009, 12, 31)), None, False) ('aggr', ) Parse: sum[period](X:discount):: >>> expr = u'sum[1:10](comp_unit:PV:discount)' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'PV'), (datetime.date(2000, 1, 1), datetime.date(2009, 12, 31)), None, True) ('aggr', ) Parse: sum(X) * 15:: >>> expr = u'sum[-1:-1](comp_unit:PV) * 15.0' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, False) ('aggr', ) ('value', 15.0) ('ari', , '*') Parse: sum(X * Y):: >>> expr = u'sum[-1:-1](comp_unit:PV:discount * comp_unit:AREA)' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('ari', , '*') ('aggr', ) Parse: sum(X / Y):: >>> expr = u'sum[-1:-1](comp_unit:X:discount / comp_unit:Y)' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'X'), (None, None), None, True) ('data', (u'comp_unit', u'Y'), (None, None), None, False) ('ari', , '/') ('aggr', ) Parse: sum(X * Y) + sum(Z):: >>> expr = u'''sum[-1:-1](comp_unit:PV:discount * comp_unit:AREA) + ... sum[1:-1](operation:cash_flow:discount)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('ari', , '*') ('aggr', ) ('op', u'cash_flow', (datetime.date(2000, 1, 1), None), None, True) ('aggr', ) ('ari', , '+') Parse: sum(X * Y):: >>> expr = u'sum[-1:-1](operation:Income:discount * comp_unit:AREA)' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('op', u'Income', (None, None), None, True) ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('ari', , '*') ('aggr', ) Parse: sum(X) >= sum(Y):: >>> expr = u'''sum[-1:-1](comp_unit:PV:discount) ge ... sum[-1:-1](comp_unit:PV:discount)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('aggr', ) ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('aggr', ) ('eq', , 'ge') Parse: (sum(X) >= sum(Y)) and (sum(Z) != 0.0):: >>> expr = u'''sum[-1:-1](comp_unit:PV:discount) ge ... sum[-1:-1](comp_unit:PV:discount) ... and ... sum[-1:-1](operation:Volume) ue 0.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('aggr', ) ('data', (u'comp_unit', u'PV'), (None, None), None, True) ('aggr', ) ('eq', , 'ge') ('op', u'Volume', (None, None), None, False) ('aggr', ) ('value', 0.0) ('eq', , 'ue') ('eq', , 'and') Parse: sum(X * Y + X * Z):: >>> expr = u'''sum[-1:-1](comp_unit:AREA * comp_unit:PV + comp_unit:AREA * ... comp_unit:PV_land)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('data', (u'comp_unit', u'PV'), (None, None), None, False) ('ari', , '*') ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('data', (u'comp_unit', u'PV_land'), (None, None), None, False) ('ari', , '*') ('ari', , '+') ('aggr', ) Parse: sum(Z * (X + Y)):: >>> expr = u'''sum[-1:-1](comp_unit:AREA * ... (comp_unit:PV + comp_unit:PV_land))''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('data', (u'comp_unit', u'PV'), (None, None), None, False) ('data', (u'comp_unit', u'PV_land'), (None, None), None, False) ('ari', , '+') ('ari', , '*') ('aggr', ) Parse: sum(X) - sum(Y) > 0: >>> expr = u'''sum[20:20](comp_unit:V) - sum[10:10](comp_unit:V) gt 0.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False) ('aggr', ) ('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('aggr', ) ('ari', , '-') ('value', 0.0) ('eq', , 'gt') Parse: sum(X) - sum(Y) > 0: >>> expr = u'''sum[20:-1](operation:Volume) - sum[10:-1](operation:Volume) gt 0.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('op', u'Volume', (datetime.date(2019, 1, 1), None), None, False) ('aggr', ) ('op', u'Volume', (datetime.date(2009, 1, 1), None), None, False) ('aggr', ) ('ari', , '-') ('value', 0.0) ('eq', , 'gt') Parse: sum(X * Z) - sum(Y * Z) > 0: >>> expr = u'''sum[20:20](comp_unit:V * comp_unit:AREA) - ... sum[10:10](comp_unit:V * comp_unit:AREA) gt 0.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('ari', , '-') ('value', 0.0) ('eq', , 'gt') Parse: sum((X + Y) * (Z + K)):: >>> expr = u'''sum[-1:-1]((comp_unit:PV + comp_unit:PV_land) * ... (comp_unit:AREA + operation:cash_flow))''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, False) ('data', (u'comp_unit', u'PV_land'), (None, None), None, False) ('ari', , '+') ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('op', u'cash_flow', (None, None), None, False) ('ari', , '+') ('ari', , '*') ('aggr', ) Parse: sum(X * Y) > sum(Z * K):: >>> expr = u'''sum[20:20](comp_unit:V * comp_unit:AREA) gt ... sum[10:10](comp_unit:V * comp_unit:AREA)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('eq', , 'gt') Parse: sum(X) - 0.05 * 1000 * 25 > 5.0:: >>> expr = u'''sum[1:-1](operation:Volume / comp_unit:AREA) - 0.05 * 1000 ... gt 5.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: ... if i[0] != 'value': ... print i ... else: ... print (i[0], '%.2f' % i[1]) ... # doctest: +ELLIPSIS ('op', u'Volume', (datetime.date(2000, 1, 1), None), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1), None), None, False) ('ari', , '/') ('aggr', ) ('value', '0.05') ('value', '1000.00') ('ari', , '*') ('ari', , '-') ('value', '5.00') ('eq', , 'gt') Parse: sum((X + Y) * Z) > 1000.0:: >>> expr = u'''sum[-1:-1]((comp_unit:PV + comp_unit:PV_land) * ... comp_unit:AREA) gt 1000.0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS ('data', (u'comp_unit', u'PV'), (None, None), None, False) ('data', (u'comp_unit', u'PV_land'), (None, None), None, False) ('ari', , '+') ('data', (u'comp_unit', u'AREA'), (None, None), None, False) ('ari', , '*') ('aggr', ) ('value', 1000.0) ('eq', , 'gt') Parse: sum(X) / sum(Y) / 0.10:: >>> expr = u'''sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) / ... sum[-1:-1](comp_unit:V) / ... 0.10''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'V'), (None, None), (('data', u'comp_unit', u'SP'), ('value', 5), ('eq', , 'eq')), False) ('aggr', ) ('data', (u'comp_unit', u'V'), (None, None), None, False) ('aggr', ) ('ari', , '/') ('value', 0.1...) ('ari', , '/') Parse: sum(X) + sum(Y) / 0.10:: >>> expr = u'''sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) + ... sum[-1:-1](comp_unit:V) / ... 0.10''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'V'), (None, None), (('data', u'comp_unit', u'SP'), ('value', 5), ('eq', , 'eq')), False) ('aggr', ) ('data', (u'comp_unit', u'V'), (None, None), None, False) ('aggr', ) ('value', 0.1...) ('ari', , '/') ('ari', , '+') Parse: (sum(X) + sum(Y)) / 0.10:: >>> expr = u'''(sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) + ... sum[-1:-1](comp_unit:V)) / ... 0.10''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'V'), (None, None), (('data', u'comp_unit', u'SP'), ('value', 5), ('eq', , 'eq')), False) ('aggr', ) ('data', (u'comp_unit', u'V'), (None, None), None, False) ('aggr', ) ('ari', , '+') ('value', 0.1...) ('ari', , '/') Parse: (sum(X) / sum(Y) > 0.05) and (sum(Z) / sum(K) < 0.15):: >>> expr = u'''(sum[-1:-1](operation:Volume[operation:SP eq 1]) / ... sum[-1:-1](operation:Volume) gt 0.05) ... and ... (sum[-1:-1](operation:Volume[operation:SP eq 1]) / ... sum[-1:-1](operation:Volume) lt 0.15)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: ... if i[0] != 'value': ... print i ... else: ... print (i[0], '%.2f' % i[1]) ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('op', u'Volume', (None, None), (('op', 'op', u'SP'), ('value', 1), ('eq', , 'eq')), False) ('aggr', ) ('op', u'Volume', (None, None), None, False) ('aggr', ) ('ari', , '/') ('value', '0.05') ('eq', , 'gt') ('op', u'Volume', (None, None), (('op', 'op', u'SP'), ('value', 1), ('eq', , 'eq')), False) ('aggr', ) ('op', u'Volume', (None, None), None, False) ('aggr', ) ('ari', , '/') ('value', '0.15') ('eq', , 'lt') ('eq', , 'and') Parse: sum(X) with simple conditions for variable X:: >>> expr = u'''sum[-1:-1](operation:Income[operation:SP eq 1]:discount)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('op', u'Income', (None, None), (('op', 'op', u'SP'), ('value', 1), ('eq', , 'eq')), True) ('aggr', ) >>> expr = u'''sum[-1:-1](operation:Income[operation:op_name eq clearcut])''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('op', u'Income', (None, None), (('op', 'op', u'op_name'), ('value', u'clearcut'), ('eq', , 'eq')), False) ('aggr', ) >>> expr = u'''sum[-1:-1](operation:Income[operation:op_group eq final_harvest ... and comp_unit:MAIN_SP eq 1])''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('combined', u'Income', (None, None), (('combined', 'op', u'op_group'), ('value', u'final_harvest'), ('eq', , 'eq'), ('combined', u'comp_unit', u'MAIN_SP'), ('value', 1), ('eq', , 'eq'), ('eq', , 'and')), False) ('aggr', ) >>> expr = u'''sum[-1:-1](operation:Income[comp_unit:MAIN_GROUP eq 1]:discount)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('combined', u'Income', (None, None), (('combined', u'comp_unit', u'MAIN_GROUP'), ('value', 1), ('eq', , 'eq')), True) ('aggr', ) Parse: sum(X) with a condition with multiple conditions for variable X:: >>> expr = u'''sum[-1:-1](operation:Income[operation:op_name eq thinning and ... operation:SP eq 1 and operation:assortment eq 1]:discount)''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('op', u'Income', (None, None), (('op', 'op', u'op_name'), ('value', u'thinning'), ('eq', , 'eq'), ('op', 'op', u'SP'), ('value', 1), ('eq', , 'eq'), ('eq', , 'and'), ('op', 'op', u'assortment'), ('value', 1), ('eq', , 'eq'), ('eq', , 'and')), True) ('aggr', ) Parse: (sum(X * Y) / sum(X * Y) - 0.1) >= 0, with some conditions:: >>> expr = u'''sum[1:1](comp_unit:V[comp_unit:MAIN_SP in (3, 4, 5, 6, 7, 9)] * ... comp_unit:AREA) / ... sum[1:1](comp_unit:V * comp_unit:AREA) - 0.1 ge 0''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: ... if i[0] != 'value': ... print i ... else: ... print (i[0], '%.2f' % i[1]) ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'V'), (datetime.date(2000, 1, 1), datetime.date(2000, 12, 31)), (('data', u'comp_unit', u'MAIN_SP'), ('value', (3, 4, 5, 6, 7, 9)), ('eq', , 'in')), False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1), datetime.date(2000, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('data', (u'comp_unit', u'V'), (datetime.date(2000, 1, 1), datetime.date(2000, 12, 31)), None, False) ('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1), datetime.date(2000, 12, 31)), None, False) ('ari', , '*') ('aggr', ) ('ari', , '/') ('value', '0.10') ('ari', , '-') ('value', '0.00') ('eq', , 'ge') Parse expressions with invalid syntax:: >>> expr = u'''comp_unit:PV[-1:-1]''' >>> c,e = parser.parse(expr, initdate, step) >>> print e (0, "Expected one of 'sum', 'mean', 'min', 'max', 'avg'") >>> expr = u'''sum[-1:-1](comp_unit:PV * 10.0)''' >>> c,e = parser.parse(expr, initdate, step) >>> print e (28, 'Expected ":"') >>> expr = u'''sum[-1:-1](comp_unit:PV * comp_unit:AREA > 1000.0''' >>> c,e = parser.parse(expr, initdate, step) >>> print e (41, 'Expected ")"') >>> expr = u'''sum[1:-1](comp_unit:V[operation:SP eq 1])''' >>> c,e = parser.parse(expr, initdate, step) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE Parse expressions using aggregation constraints :: >>> expr = u'sum[10:10](comp_unit:PV)' >>> c,e = parser.parse(expr, initdate, step, ('sum','min','max')) >>> for i in c: print i # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ('data', (u'comp_unit', u'PV'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False) ('aggr', ) Test the error handling in the case of invalid aggregation function :: >>> c,e = parser.parse(expr, initdate, step, ('min','max')) >>> print e (0, "Invalid aggregator function 'sum'")