Source code for ClearMap.IO.NPY

# -*- coding: utf-8 -*-
"""
NPY
===

IO interface to numpy arrays.
"""
__author__    = 'Christoph Kirst <christoph.kirst.ck@gmail.com>'
__license__   = 'GPLv3 - GNU General Pulic License v3 (see LICENSE.txt)'
__copyright__ = 'Copyright © 2020 by Christoph Kirst'
__webpage__   = 'http://idisco.info'
__download__  = 'http://www.github.com/ChristophKirst/ClearMap2'


import numpy as np

import ClearMap.IO.Source as src
#import ClearMap.IO.FileUtils as fu

###############################################################################
### Source classe
###############################################################################

[docs] class Source(src.Source): """Numpy array source.""" def __init__(self, array = None, shape = None, dtype = None, order = None, name = None): """Numpy source class construtor. Arguments --------- array : array The underlying data array of this source. """ super(Source, self).__init__(name=name); self._array = _array(shape=shape, dtype=dtype, order=order, array=array); def __getattr__(self, name): #numpy attributes if name != '_array' and hasattr(self, '_array') and hasattr(self._array, name): return getattr(self._array, name); else: raise AttributeError('Not such attribute %r!' % name); #def __del__(self): # self._array = None; @property def name(self): return "Numpy-Source"; @property def array(self): """The underlying data array. Returns ------- array : array The underlying data array of this source. """ return self._array; @array.setter def array(self, value): self._array = _array(value); @property def shape(self): """The shape of the source. Returns ------- shape : tuple The shape of the source. """ return self._array.shape; @shape.setter def shape(self, value): self._array.shape = value; @property def dtype(self): """The data type of the source. Returns ------- dtype : dtype The data type of the source. """ return self._array.dtype; @dtype.setter def dtype(self, value): self._array = np.asarray(self._array, dtype=value) @property def order(self): """The order of how the data is stored in the source. Returns ------- order : str Returns 'C' for C contigous and 'F' for fortran contigous, None otherwise. """ return order(self.array); @order.setter def order(self, value): self._array = np.asarray(self._array, order = value) @property def element_strides(self): """The strides of the array elements. Returns ------- strides : tuple Strides of the array elements. Note ---- The strides of the elements module itemsize instead of bytes. """ return tuple(s // self._array.itemsize for s in self._array.strides) @property def offset(self): """The offset of the memory map in the file. Returns ------- offset : int Offset of the memeory map in the file. """ if self._array.base is not None: return np.byte_bounds(self._array)[0] - np.byte_bounds(self._array.base)[0]; else: return 0; ### Parallel processing
[docs] def as_virtual(self): #TODO: convert to shared memory array ? -> needs to be implmented to make block processing work for in memory numpay arrays ! return self;
[docs] def as_real(self): return self;
[docs] def as_buffer(self): return self._array;
### Data def __getitem__(self, *args): return self.array.__getitem__(*args); def __setitem__(self, *args): self.array.__setitem__(*args);
#class Array(np.ndarray): # """Array wrapper around numpy ndarray.""" # # def __new__(cls, array): # obj = np.asarray(array).view(cls) # #obj.order = order(obj); # return obj # # def __array_finalize__(self, obj): # if obj is None: return # #self.order = getattr(obj, 'order', None) # # def name(self): # return "Source-Numpy"; # # def array(self): # return self.view(np.ndarray); # # @property # def order(self): # return order(self); # # @property # def array_strides(self): # return tuple(np.array(self.strides, dtype = int) / self.itemsize); # # def __str__(self): # if self.shape is not None: # shape = '%r' % ((self.shape,)); # else: # shape = ''; # # if self.dtype is not None: # dtype = '[%s]' % self.dtype; # else: # dtype = ''; # # if order(self) is not None: # _order = '|%s|' % order(self); # else: # _order = ''; # # array = super(Array, self).__str__(); # if len(array) > 100: # e = array[100:].find('\n'); # if e != -1: # array = array[:100 + e] + '...'; # if len(array) > 0: # array = '\n' + array; # else: # array = ''; # # return 'Array' + shape + dtype + _order + array # # def __repr__(self): # return self.__str__(); ############################################################################### ### Functionality ###############################################################################
[docs] def order(array): """Returns the contigous order of an array. Arguments --------- array : ndarray Returns ------- order : 'C', 'F', None """ if isinstance(array, src.Source): return array.order; elif isinstance(array, np.ndarray): if array.flags['C_CONTIGUOUS']: return 'C' elif array.flags['F_CONTIGUOUS']: return 'F' else: return None; else: return None;
############################################################################### ### IO Interface ###############################################################################
[docs] def is_numpy(source): if isinstance(source, (Source, np.ndarray, list, tuple)): return True; #elif isinstance(source, str): # and fu.file_extension(source) == 'npy': # return True; else: return False;
[docs] def read(source, slicing = None, as_source = None, as_array = None, processes = None, **kwargs): if isinstance(source, (list, tuple)): source = np.array(source); if isinstance(source, Source): if slicing is not None: source = source.__getitem__(slicing); if as_array: return source.array else: return source; elif isinstance(source, np.ndarray): if slicing is not None: source = source.__getitem__(slicing); if as_source: return Source(array = source); else: return source; # elif isinstance(source, str): # and fu.file_extension(source) == 'npy': # source = np.load(source); # if slicing is not None: # source = source.__getitem__(slicing); # if as_source: # return Source(array = source); # else: # return source; else: raise ValueError('The source is not a valid numpy source!')
#TODO: add processes keyword for parallel writing
[docs] def write(sink, data, slicing = None, **kwargs): if slicing is None: slicing = (); if sink is None: return data.__getitem__(slicing); if isinstance(sink, (src.Source, np.ndarray)): sink.__setitem__(slicing, data); return sink; # elif isinstance(sink, str): #and fu.file_extension(sink) == 'npy' # if slicing != (): # if not fu.is_file(sink): # raise ValueError('Cannot write slice to a not existing file %s!' % sink); # memmap = np.lib.format.open_memmap(sink); # memmap.__setitem__(slicing, data); # else: # np.save(sink, data); # return sink; else: raise ValueError('The sink is not a valid numpy sink!')
[docs] def create(shape = None, dtype = None, order = None, array = None, as_source = True, **kwargs): """Create a numpy array. Arguments --------- shape : tuple or None The shape of the memory map to create. dtype : dtype The data type of the memory map. order : 'C', 'F', or None The contiguous order of the memmap. array : array, Source or None Optional source with data to fill the numpy array with. as_source : bool If True, return as Source class. Returns ------- array : np.array The numpy array. Note ---- By default numpy arrays are initialized as fortran contiguous if order is None. """ array = _array(shape=shape, dtype=dtype, order=order, array=array) if as_source: return Source(array=array); else: return array;
############################################################################### ### Helpers ############################################################################### def _order(array): return order(array); def _array(shape = None, dtype = None, order = None, array = None): """Create a numpy array. Arguments --------- shape : tuple or None The shape of the memory map to create. dtype : dtype The data type of the memory map. order : 'C', 'F', or None The contiguous order of the memmap. array : array, Source or None Optional source with data to fill the memory map. Returns ------- array : np.ndarray The array. """ if isinstance(array, (list, tuple)): array = np.asarray(array, order=order, dtype=dtype); if isinstance(array, np.ndarray): shape = shape if shape is not None else array.shape; dtype = dtype if dtype is not None else array.dtype; order = order if order is not None else _order(array); if shape != array.shape: raise ValueError('Shape %r and array shape %r mismatch!' % (shape, array.shape)); if dtype != array.dtype or order != _order(array): array = np.asarray(array, order=order, dtype=dtype); else: if shape is None: raise ValueError('Cannot create array without shape!'); array = np.zeros(shape, dtype=dtype, order=order); return array; ############################################################################### ### Tests ############################################################################### def _test(): import numpy as np import ClearMap.IO.NPY as npy; #reload(npy); s = npy.Source(array=np.zeros((5,7))); print(s); import ClearMap.IO.Slice as slc t = slc.Slice(source= s, slicing= (1,)); print(t); v = t.as_virtual() print(v); x = np.ones(250*1000*1000) xs = npy.Source(array=x); print(xs) del x del xs