Navigation

Table Of Contents

Previous topic

db.py

Next topic

importops.py

This Page

importdata.py

class DataImporter(object):

Importing data from ‘inlined’ format; i.e. the data for each level in one file:

>>> from simo.input.importdata import DataImporter
>>> execfile('input/test/mock4importdata.py')
>>> #from simo.input.test.mock4importdata import *
>>> imp = DataImporter(inputdb, mapping, importdate,
...                    logger, logname, lexicon, 100)
>>> imp.import_data('inlined', [inline], 'simulation')
...     # doctest: +ELLIPSIS
...     # doctest: +NORMALIZE_WHITESPACE
Called Logger.log_message('testlog', 'info', 'Importing data...')
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'simulation': [(datetime.date(2009, 1, 6), {'oid': 'simulation',
    'values': [], 'id': 'simulation', 'parent id': None})]},
    0,
    0)
Called DataDB.add_data_from_dictionary(
    {'comp_unit': [(datetime.date(2009, 1, 6), {'oid': 'stand1',
    'values': [('DEV_CLASS', 1), ('ORIG_DC', 1.0), ('MAIN_GROUP', 1),
     ('SOMETHING_ELSE', 99), ('Inventory_date', 733413),
     ('USE_RESTRICTION_SILVIC', '0'), ('USE_RESTRICTION_HARVEST', '0')],
     'id': 'stand1', 'parent id': 'simulation'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'simulation': [(datetime.date(2009, 1, 6), {'oid': 'simulation',
     'values': [], 'id': 'simulation', 'parent id': None})]},
    0,
    0)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 1, 6), {'oid': 'stratum1_1',
     'values': [('BA', 200.0), ('BT', 'test')], 'id': 'stand1-stratum1_1',
      'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'simulation': [(datetime.date(2009, 1, 6), {'oid': 'simulation',
     'values': [], 'id': 'simulation', 'parent id': None})]},
    0,
    0)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 1, 6), {'oid': 'stratum1_2',
     'values': [('BA', 22.0), ('BT', 'piece')], 'id': 'stand1-stratum1_2',
      'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called Logger.log_message(
    'testlog',
    'info',
    'In total 2 simulation units processed')
Called Logger.log_message(
    'testlog',
    'info',
    'In total 1 simulation units imported')
False

Importing data in ‘by_level’ format; i.e., each data level has its’ own file:

>>> imp.import_data('by_level', by_level, 'simulation')
...     # doctest: +NORMALIZE_WHITESPACE
...     # doctest: +ELLIPSIS
Called Logger.log_message('testlog', 'info', 'Importing data...')
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'comp_unit': [(datetime.date(2009, 1, 1), {'oid': 'stand1',
    'values': [('MAIN_GROUP', 1), ('SOMETHING_ELSE', 99),
    ('Inventory_date', 733408), ('USE_RESTRICTION_SILVIC', '0'),
    ('USE_RESTRICTION_HARVEST', '0')], 'id': 'stand1', 'parent id':
    'simulation'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'comp_unit': [(datetime.date(2009, 12, 31), {'oid': 'stand2',
    'values': [('MAIN_GROUP', 1), ('SOMETHING_ELSE', 99),
    ('Inventory_date', 733772), ('USE_RESTRICTION_SILVIC', '0'),
    ('USE_RESTRICTION_HARVEST', '0')], 'id': 'stand2', 'parent id':
    'simulation'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 1, 1), {'oid': 'stratum1_1',
     'values': [('BA', 200.0), ('BT', 'oh')], 'id': 'stand1-stratum1_1',
      'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 1, 1), {'oid': 'stratum1_2',
     'values': [('BA', 22.0), ('BT', 'which')], 'id': 'stand1-stratum1_2',
      'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 12, 31), {'oid': 'stratum2_1',
     'values': [('BA', 31.0), ('BT', 'is')], 'id': 'stand2-stratum2_1',
      'parent id': 'stand2'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 12, 31), {'oid': 'stratum2_2',
     'values': [('BA', 1.0), ('BT', 'infact')], 'id': 'stand2-stratum2_2',
     'parent id': 'stand2'})]},
    0,
    0)
Called Logger.log_message(
    'testlog',
    'info',
    'In total 2 simulation units processed')
Called Logger.log_message(
    'testlog',
    'info',
    'In total 2 simulation units imported')
False

>>> imp.errors
set([])

With skipfirst. If used like here when the first row shoudn’t really be skipped, results in orphan lower data level objects in the database. Also tests id generation; the strata for stand2 have missing ids, so they’ll get ids 1 and 2:

>>> imp.import_data('inlined', [inline2], 'simulation', skip_first=True)
...      # doctest: +ELLIPSIS
...      # doctest: +NORMALIZE_WHITESPACE
Called Logger.log_message('testlog', 'info', 'Importing data...')
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'simulation': [(None, {'oid': 'simulation', 'values': [],
    'id': 'simulation', 'parent id': None})]},
    0,
    0)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(None, {'oid': 'stratum1_1', 'values':
    [('BA', 200.0), ('BT', 'pretty')], 'id': 'stratum1_1',
    'parent id': None})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation',
distinct=True)
Called DataDB.add_data_from_dictionary(
    {'simulation': [(None, {'oid': 'simulation', 'values': [],
    'id': 'simulation', 'parent id': None})]},
    0,
    0)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(None, {'oid': 'stratum1_2', 'values':
    [('BA', 22.0), ('BT', 'frekin')], 'id': 'stratum1_2',
    'parent id': None})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation',
distinct=True)
Called DataDB.add_data_from_dictionary(
    {'comp_unit': [(datetime.date(2009, 12, 31), {'oid': 'stand2',
    'values': [('MAIN_GROUP', 1), ('SOMETHING_ELSE', 99),
    ('Inventory_date', 733772), ('USE_RESTRICTION_SILVIC', '0'),
    ('USE_RESTRICTION_HARVEST', '0')], 'id': 'stand2', 'parent id':
    'simulation'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation',
distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 12, 31), {'oid': '1', 'values':
    [('BA', 31.0), ('BT', 'hmm')], 'id': 'stand2-1', 'parent id':
    'stand2'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation',
distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 12, 31), {'oid': '2',
     'values': [('BA', 1.0), ('BT', 'wait')], 'id': 'stand2-2',
      'parent id': 'stand2'})]},
    0,
    0)
Called Logger.log_message(
    'testlog',
    'info',
    'In total 1 simulation units processed')
Called Logger.log_message(
    'testlog',
    'info',
    'In total 1 simulation units imported')
False

>>> imp.errors
set([])

Specifying a separator to be used instead of the default whitespace:

>>> imp.import_data('inlined', [inline3], 'simulation', separator=';')
...      # doctest: +ELLIPSIS
...      # doctest: +NORMALIZE_WHITESPACE
Called...
    'In total 2 simulation units imported')
False

By level import for only one, not top level, level with the given data date:

>>> from datetime import date
>>> data_date = date(2009, 5, 6)
>>> imp.import_data('by_level', by_level2, 'simulation', level_ind=[1],
...                 data_date=data_date, empty_database=False)
...      # doctest: +ELLIPSIS
...      # doctest: +NORMALIZE_WHITESPACE
Called Logger.log_message('testlog', 'info', 'Importing data...')
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 5, 6),
    {'oid': 'stratum1_1', 'values': [('BA', 200.0), ('BT', 'oh')],
    'id': 'stand1-stratum1_1', 'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 5, 6),
    {'oid': 'stratum1_2', 'values': [('BA', 22.0), ('BT', 'which')],
    'id': 'stand1-stratum1_2', 'parent id': 'stand1'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 5, 6),
    {'oid': 'stratum2_1', 'values': [('BA', 31.0), ('BT', 'is')],
    'id': 'stand2-stratum2_1', 'parent id': 'stand2'})]},
    0,
    0)
Called DataDB.get_data_from_level('data', ['date'], 'simulation', distinct=True)
Called DataDB.add_data_from_dictionary(
    {'stratum': [(datetime.date(2009, 5, 6),
    {'oid': 'stratum2_2', 'values': [('BA', 1.0), ('BT', 'infact')],
    'id': 'stand2-stratum2_2', 'parent id': 'stand2'})]},
    0,
    0)
Called Logger.log_message(
    'testlog',
    'info',
    'In total 0 simulation units processed')
Called Logger.log_message(
    'testlog',
    'info',
    'In total 0 simulation units imported')
False

def _construct_unique_id(self, level, oid, parent_id):

Construct a unique id for an object:

>>> imp._construct_unique_id(0, '1', 'simulation')
'1'
>>> imp._construct_unique_id(1, '1', 'stratum1_2')
Called Logger.log_message(
    'testlog',
    'error',
    "No parent path available from 'stratum' to 'stratum'!")
'stratum1_2-1'
>>> imp._construct_unique_id(1, '1', 'stand1')
'stand1-1'

def _parse_date(self, datestr):

Parse a date string into a datetime object.

>>> dates = ['230209', '23.07.09', '23-07-09', '23/07/09',
...          '23072009', '23.07.2009', '23-07-2009', '23/07/2009',
...          '2009-07-23', 'fail']
>>> [imp._parse_date(date) for date in dates] #doctest: +NORMALIZE_WHITESPACE
Called Logger.log_message('testlog', 'error', "Invalid date format 'fail'")
[datetime.date(2009, 2, 23), datetime.date(2009, 7, 23),
 datetime.date(2009, 7, 23), datetime.date(2009, 7, 23),
 datetime.date(2009, 7, 23), datetime.date(2009, 7, 23),
 datetime.date(2009, 7, 23), datetime.date(2009, 7, 23),
 datetime.date(2009, 7, 23), None]

Navigation

© Copyright 2009, Simosol Oy. Last updated on Nov 04, 2010. Created using Sphinx 0.6.5.