Fix Integer Overflow

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001

activitysim/abm/tables/skims.py

@@ -15,6 +15,8 @@
 import multiprocessing
 
 from collections import OrderedDict
+from functools import reduce
+from operator import mul
 
 import numpy as np
 import openmatrix as omx
@@ -90,7 +92,7 @@ def get_skim_info(omx_file_path, tags_to_load=None):
 
  if MAX_BLOCK_BYTES:
  max_block_items = MAX_BLOCK_BYTES // np.dtype(skim_dtype).itemsize
- max_skims_per_block = max_block_items // np.prod(omx_shape)
+ max_skims_per_block = max_block_items // multiply_large_numbers(omx_shape)
  else:
  max_skims_per_block = num_skims
 
@@ -149,14 +151,14 @@ def block_name(block):
 def buffers_for_skims(skim_info, shared=False):
 
  skim_dtype = skim_info['dtype']
- omx_shape = [np.float64(x) for x in skim_info['omx_shape']]
+ omx_shape = skim_info['omx_shape']
  blocks = skim_info['blocks']
 
  skim_buffers = {}
  for block_name, block_size in iteritems(blocks):
 
  # buffer_size must be int (or p2.7 long), not np.int64
- buffer_size = int(np.prod(omx_shape) * block_size)
+ buffer_size = int(multiply_large_numbers(omx_shape) * block_size)
 
  csz = buffer_size * np.dtype(skim_dtype).itemsize
  logger.info("allocating shared buffer %s for %s (%s) matrices (%s)" %
@@ -191,7 +193,7 @@ def skim_data_from_buffers(skim_buffers, skim_info):
  for block_name, block_size in iteritems(blocks):
  skims_shape = omx_shape + (block_size,)
  block_buffer = skim_buffers[block_name]
- assert len(block_buffer) == int(np.prod(skims_shape))
+ assert len(block_buffer) == int(multiply_large_numbers(skims_shape))
  block_data = np.frombuffer(block_buffer, dtype=skim_dtype).reshape(skims_shape)
  skim_data.append(block_data)
 
@@ -261,6 +263,10 @@ def skim_dict(data_dir, settings):
  return skim_dict
 
 
+def multiply_large_numbers(list_of_numbers):
+ return reduce(mul, list_of_numbers)
+
+
 @inject.injectable(cache=True)
 def skim_stack(skim_dict):
 

activitysim/abm/test/test_skims.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+from future.utils import iteritems
+
+import numpy as np
+import pytest
+
+from activitysim.abm.tables import skims
+
+
+@pytest.fixture(scope="session")
+def matrix_dimension():
+ return 5922
+
+
+@pytest.fixture(scope="session")
+def num_of_matrices():
+ return 845
+
+
+@pytest.fixture(scope="session")
+def skim_info(num_of_matrices, matrix_dimension):
+ time_periods = ['EA', 'AM', 'MD', 'PM', 'NT']
+
+ omx_keys = OrderedDict()
+ omx_key1_block_offsets = OrderedDict()
+ omx_block_offsets = OrderedDict()
+ omx_blocks = OrderedDict()
+ omx_blocks['skim_arc_skims_0'] = num_of_matrices
+
+ for i in range(0, num_of_matrices + 1):
+ key1_name = 'm{}'.format(i // len(time_periods) + 1)
+ time_period = time_periods[i % len(time_periods)]
+
+ omx_keys[(key1_name, time_period)] = '{}__{}'.format(key1_name, time_period)
+ omx_block_offsets[(key1_name, time_period)] = (0, i)
+
+ if 0 == i % len(time_periods):
+ omx_key1_block_offsets[key1_name] = (0, i)
+
+ skim_info = {
+ 'omx_name': 'arc_skims',
+ 'omx_shape': (matrix_dimension, matrix_dimension),
+ 'num_skims': num_of_matrices,
+ 'dtype': np.float32,
+ 'omx_keys': omx_keys,
+ 'key1_block_offsets': omx_key1_block_offsets,
+ 'block_offsets': omx_block_offsets,
+ 'blocks': omx_blocks
+ }
+
+ return skim_info
+
+
+def test_multiply_large_numbers(skim_info, num_of_matrices, matrix_dimension):
+ omx_shape = skim_info['omx_shape']
+ blocks = skim_info['blocks']
+
+ for block_name, block_size in iteritems(blocks):
+ # If overflow, this number will go negative
+ assert int(skims.multiply_large_numbers(omx_shape) * block_size) == \
+ num_of_matrices * matrix_dimension ** 2
+
+
+def test_multiple_large_floats():
+ calculated_value = skims.multiply_large_numbers([6205.1, 5423.2, 932.4, 15.4])
+ actual_value = 483200518316.9472
+ assert abs(calculated_value - actual_value) < 0.0001