Source code for ClearMap.IO.Slice

# -*- coding: utf-8 -*-
"""
Slice
=====

This module provides basic handling of slicing of sources.

The main functionality is to virtually slice an array and return its 
expected shape and order. Virtual slices can also be used as handles with 
low communication overhead in parallel processing.

Example
-------
>>> import numpy as np
>>> import ClearMap.IO.IO as io
>>> source = io.source(np.random.rand(30,40))
>>> sliced = io.slc.Slice(source, slicing=(slice(None), slice(10,20)))
>>> sliced
Sliced-Numpy-Source(30, 10)[float64]

>>> sliced.base
Numpy-Source(30, 40)[float64]|C|
"""
__author__    = 'Christoph Kirst <christoph.kirst.ck@gmail.com>'
__license__   = 'GPLv3 - GNU General Pulic License v3 (see LICENSE.txt)'
__copyright__ = 'Copyright © 2020 by Christoph Kirst'
__webpage__   = 'http://idisco.info'
__download__  = 'http://www.github.com/ChristophKirst/ClearMap2'


import numbers
import numpy as np

import ClearMap.IO.Source as src


###############################################################################
### Source class
###############################################################################

[docs] class Slice(src.Source): """A virtual slice of a source.""" def __init__(self, source = None, slicing = None, name = None): """Slice class construtor. Arguments --------- source : class The underlying source of this slice. slicing : int, slice, list of slices or None The slice specification. """ super(Slice, self).__init__(name=name); self._source = source; self._slicing = slicing; @property def name(self): return 'Sliced-' + self.source.name; @property def source(self): """The source of this sliced source. Returns ------- source : Source class The source of this slice. """ return self._source; @source.setter def source(self, source): self.__init__(source = source, slicing = self.slicing); @property def slicing(self): """The slice spcification of the source. Returns ------- slicing : slice, list or None Returns the slice specification of this sliced source. """ return self._slicing; @slicing.setter def slicing(self, value): self.__init__(source = self.source, slicing = value); @property def shape(self): """The shape of the source. Returns ------- shape : tuple The shape of the source. """ return sliced_shape(self.slicing, self.source.shape); @property def dtype(self): """The data type of the source. Returns ------- dtype : dtype The data type of the source. """ return self.source.dtype; @property def order(self): """The order of how the data is stored in the source. Returns ------- order : str Returns 'C' for C contigous and 'F' for fortran contigous, None otherwise. """ return sliced_order(self.slicing, self.source.order, self.source.shape); @property def strides(self): """The strides of the data source. Returns ------- strides : tuple of ints The strides of the sliced source. """ return sliced_strides(self.slicing, self.source.strides); @property def element_strides(self): """The strides of the data source. Returns ------- strides : tuple of ints The strides of the sliced source. """ return sliced_strides(self.slicing, self.source.element_strides); @property def offset(self): """The offset of the memory map in the file. Returns ------- offset : int Offset of the memeory map in the file. """ return self.source.offset + sliced_offset(self.slicing, self.source.strides); # @property # def memory(self): # """The memory type of the source. # # Returns # ------- # memory : str # Returns 'shared' for a shared memory buffer, 'memmap' for a memory map to a file, None otherwise. # """ # return self.source.memory; @property def location(self): """The location of the source's data. Returns ------- location : str Returns the location of the data source or None if there is none. """ return self.source.location; @property def unpacked_slicing(self): """The unpacked slicing specification. Returns ------- slicing : tuple Returns the unpacked slice spcification of this sliced source. """ return unpack_slicing(self.slicing, self.source.ndim); @property def base_shape(self): """The shape of the underlying base source in case of nested slicing. Returns ------- shape : tuple Returns the shape for the base of this source. """ if isinstance(self.source, Slice): return self.source.base_shape; else: return self.source.shape; @property def base_slicing(self): """The direct slicing specification of the underlying base source in case of nested slicing. Returns ------- slicing : tuple Returns the slice spcification for the base of this source. """ if isinstance(self.source, Slice): return sliced_slicing(self.slicing, self.source.base_slicing, self.base_shape); else: return self.slicing; @property def base(self): """The underlying base source in case of nested slicing. Returns ------- source : Source Returns the underlying base source of this sliced source. """ if isinstance(self.source, Slice): return self.source.base; else: return self.source; @property def position(self): """Returns the indices of the lower corner of this slice in the underlying source. Returns ------- position : tuple of int The coordinates of the lower corner of this slice within the source. """ return tuple(sl.indices(s)[0] for sl,s in zip(self.slicing, self.source.shape)); @property def base_position(self): """Returns the indices of the lower corner of this slice in the underlying base source. Returns ------- position : tuple of int The coordinates of the lower corner of this slice within the base source. """ return tuple(sl.indices(s)[0] for sl,s in zip(self.base_slicing, self.base.shape)); @property def lower(self): """Returns the indices of the lower corner of this slice in the underlying source. Returns ------- position : tuple of int The coordinates of the lower corner of this slice within the source. """ return self.position; @property def base_lower(self): """Returns the indices of the lower corner of this slice in the underlying base source. Returns ------- position : tuple of int The coordinates of the lower corner of this slice within the base source. """ return self.base_position; @property def upper(self): """Returns the indices of the upper corner of this slice in the underlying source. Returns ------- position : tuple of int The coordinates of the upper corner of this slice within the source. """ return tuple(sl.indices(s)[1] for sl,s in zip(self.slicing, self.source.shape)); @property def base_upper(self): """Returns the indices of the upper corner of this slice in the underlying base source. Returns ------- position : tuple of int The coordinates of the upper corner of this slice within the base source. """ return tuple(sl.indices(s)[1] for sl,s in zip(self.base_slicing, self.base.shape)); @property def array(self): """The sliced array. Returns ------- array : array An array representing the slice, if the underling source has an slicable array property, else None. """ return self.source.__getitem__(self.slicing); #if hasattr(self.source, 'array'): # array = self.source.array; # if array is not None: # return array[self.slicing]; # else: # return None; #else: # return None; #def read(self, source = None, shape = None, dtype = None, order = None, memory = None): def __getitem__(self, slicing): slicing = sliced_slicing(slicing, self.slicing, self.source.shape); return self.source.__getitem__(slicing); def __setitem__(self, slicing, data): slicing = sliced_slicing(slicing, self.slicing, self.source.shape); self.source.__setitem__(slicing, data);
[docs] def as_virtual(self): """Returns a virtual handle to this source striped of any big array data useful for parallel processing. Returns ------- source : Source class The source class with out any cached array data. Note ---- The slicing structure is kept here to be able to appropiately take slices in a source when processsing in parallel. """ return Slice(source = self.source.as_virtual(), slicing = self.slicing);
[docs] def as_real(self): return Slice(source = self.source.as_real(), slicing = self.slicing);
[docs] def as_buffer(self): return self.array;
############################################################################### ### Functionality ############################################################################### allow_index_arrays = False; """Default value to allow index arrays in slicing. Note ---- If True, allows indexing to also be integer or boolean arrays. In numpy this type of indexing triggers copying of the array and the sliced array is not a view into the original array. """
[docs] def slice_to_range(slicing, shape = None): """Transforms a slice object to a range. Arguments --------- slicing : slice A sinlge slice class. shape : tupe of ints or None The shape of the source, if None, try to determine from slice. Returns ------- range : range The range corrresponding to the slice. """ if not isinstance(slicing, slice): raise ValueError('A slice is expected!'); if shape is not None: return np.arange(*slicing.indices(shape)); elif slicing.stop is not None: return np.arange(*slicing.indices(slicing.stop)); else: raise ValueError('No way to determine the range from the slice %r without a source shape!' % slicing);
[docs] def unpack_slicing(slicing, ndim): """Convert slice specification to a slice specification that matches the dimension of the sliced array. Arguments --------- slicing : object The slice specification. ndim : int The dimension of the source to slice. Returns ------- slicing : object The full slice specification. """ if not isinstance(slicing, tuple): slicing = (slicing,) slicing = list(slicing) n_no_newaxis = len([s for s in slicing if not (s is np.newaxis or s is None)]); is_ellipsis = [s is Ellipsis for s in slicing]; n_ellipsis = np.sum(is_ellipsis) if n_ellipsis > 1: raise IndexError('Only a single ellipsis allowed in a slice specification, found %d!' % np.sum(is_ellipsis)) elif n_ellipsis == 1 and n_no_newaxis - 1 >= ndim: slicing.pop(is_ellipsis.index(True)); n_no_newaxis -= 1 if n_no_newaxis > ndim: raise IndexError('Slice specification has more dimensions %d than array %d.' % (n_no_newaxis, ndim)); left = []; right = []; take_from_left = True while slicing: if take_from_left: next_s = slicing.pop(0) list_s = left; else: next_s = slicing.pop(-1) list_s = right; if next_s is Ellipsis: next_s = slice(None) take_from_left = not take_from_left; list_s.append(next_s) middle = [slice(None)] * (ndim - n_no_newaxis); return tuple(left + middle + right[::-1])
[docs] def simplify_slicing(slicing, ndim = None): """Simplifies slice specification to avoid fancy indexing if possible. Arguments --------- slicing : object The slice specification. ndim : int The dimension of the source to slice. Returns ------- slicing : object The full slice specification. """ if not isinstance(slicing, tuple): slicing = (slicing,) if ndim is not None: slicing = unpack_slicing(slicing, ndim); simple = []; for s in slicing: s = _standard_slice(s) if isinstance(s, np.ndarray): if s.dtype == bool: s = np.where(s)[0]; if len(s) == 0: simple.append(slice(0, 0)); continue; elif len(s) == 1 : simple.append(slice(s[0], s[0]+1)); continue; else: step = np.unique(np.diff(s)); if len(step) == 1: simple.append(slice(s[0], s[-1] + 1, step[0])); continue; simple.append(s); return tuple(simple);
[docs] def is_view(slicing): """Returns True if the slicing results in a view of the original array. Arguments --------- slicing : object The slice specification. ndim : int The dimension of the source to slice. Returns ------- is_view : bool True if the sliced array is a view. """ for s in slicing: if not isinstance(s, (slice, numbers.Integral)) and not (s is Ellipsis or s is None or s is np.newaxis): return False; return True;
[docs] def is_trivial(slicing): """Returns True if the slicing is not generating a real sub-slice. Arguments --------- slicing : object The slice specification. shape : tuple of ints or None. If not None assume this shape for the array to be sliced. Returns ------- is_trivial : bool True if the sliced array is changed form the original one. """ if slicing is None: return True; for s in slicing: if s is Ellipsis: continue; elif isinstance(s, slice) and s.start is None and s.stop is None and s.step is None: continue; else: return False; return True;
[docs] def sliced_ndim(slicing, ndim, allow_index_arrays = allow_index_arrays): """Returns the dimension of a slicing of an array with given dimension. Arguments --------- slicing : object Slice specification. ndim : int Diemnsion of the array. Returns ------- ndim : int The dimension of the sliced array. """ slicing = unpack_slicing(slicing, ndim); d = 0; index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; for s in slicing: s = _standard_slice(s); if isinstance(s, int): continue; elif isinstance(s, slice): d += 1; elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d) d += 1; elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) d += 1; elif s is np.newaxis or s is None: d += 1; else: raise IndexError('Invalid indexing object %r' % s); return d;
[docs] def sliced_shape(slicing, shape, allow_index_arrays = allow_index_arrays): """Returns the shape that results from slicing. Arguments --------- slicing : object Slice specification. shape : tuple Shape of the original array. Returns ------- shape : tuple The shape of the sliced array. Note ---- Fancy indexing is not supported. """ if shape is None: return None; slicing = unpack_slicing(slicing, len(shape)); sliced = [] d = -1 index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): if s >= shape[d] or -s > shape[d]: raise IndexError('Index out of range in dimension %d!' % d) continue elif isinstance(s, slice): start, stop, step = s.indices(shape[d]); sliced.append((stop - start - 1) // step + 1); elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); if len(s) != shape[d]: raise IndexError('The boolean indexing has different shape %d than the source %d in dimension %d!' % (len(s), shape[d], d)); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d) sliced.append(np.sum(s)) elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) if np.any(s >= shape[d]) or np.any(-s > shape[d]): raise IndexError('Index out of range in dimension %d!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) sliced.append(len(s)) elif s is np.newaxis or s is None: d -= 1 sliced.append(1) else: raise IndexError('Invalid indexing object %r' % s); return tuple(sliced);
[docs] def sliced_order(slicing, order, shape, allow_index_arrays =allow_index_arrays): """Returns the contiguous order of a sliced array. Arguments --------- slicing : object The slice specification. order : 'C', 'F' or None The order of the source to be sliced. shape : tuple of ints The shape of the source to be sliced. Returns ------- order : 'C', 'F' or None The order of the sliced source. """ if order is None: return None; if shape is None: return None; slicing = unpack_slicing(slicing, len(shape)); if order == 'F': slicing = slicing[::-1]; shape = shape[::-1]; #check order index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; is_subslice = False; is_initial = True; d = -1; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): if s >= shape[d] or -s > shape[d]: raise IndexError('Index out of range in dimension %d!' % d) if is_initial: continue; else: if is_subslice: return None; is_subslice = True; continue; elif isinstance(s, slice): if s == slice(None): is_subslice = True; is_initial = False; continue; start, stop, step = s.indices(shape[d]); size = (stop - start - 1) // step + 1; if size == 1: if is_initial: continue; else: if is_subslice: return None; is_subslice = True; continue; if step > 1: return None; if size == shape[d]: is_initial = False; is_subslice = True; continue; else: if is_subslice: return None; else: is_initial = False; is_subslice = True; continue; elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); if len(s) != shape[d]: raise IndexError('The boolean indexing has different shape %d than the source %d in dimension %d!' % (len(s), shape[d], d)); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d); size = np.sum(s); if size == 1: if is_initial: continue; else: if is_subslice: return None; is_subslice = True; continue; elif size == shape[d]: is_initial = False; is_subslice = True; continue; else: if is_subslice: return None; else: is_initial = False; is_subslice = True; continue; elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) if np.any(s >= shape[d]) or np.any(-s > shape[d]): raise IndexError('Index out of range in dimension %d!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) size = len(s); if size == 1: if is_initial: continue; else: if is_subslice: return None; is_subslice = True; continue; elif size == shape[d]: is_initial = False; is_subslice = True; continue; else: if is_subslice: return None; else: is_initial = False; is_subslice = True; continue; elif s is np.newaxis or s is None: continue; else: raise IndexError('Invalid indexing object %r' % s); return order;
[docs] def sliced_offset(slicing, strides, shape = None, allow_index_arrays = allow_index_arrays): """Returns the offset to the first element of the slicing into a buffer with given strides. Arguments --------- slicing : object Slice specification. strides : tuple Strides of the array. Returns ------- offset : int Offset into the sliced array. """ slicing = unpack_slicing(slicing, len(strides)); offset = 0; index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; d = -1; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): if s < 0: if shape is None: raise IndexError('Cannot determine offset without shape!'); s = shape[d] + s; if s < 0: raise IndexError('Index out of bounds in dimension %d!' % d) offset += s * strides[d]; elif isinstance(s, slice): s = (s.start or 0); if s < 0: if shape is None: raise IndexError('Cannot determine offset without shape!'); s = shape[d] + s; s = 0 if s < 0 else s; offset += s * strides[d]; elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d) start = np.where(s)[0]; if len(start) == 0: raise IndexError('There is not True value in boolean array slicing in dimension %d!' % d) offset += start[0] * strides[d]; elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) if len(s) == 0: raise IndexError('There is no index in array slicing in dimension %d!' % d) s = s[0]; if s < 0: if shape is None: raise IndexError('Cannot determine offset without shape!'); s = shape[d] + s; if s < 0: raise IndexError('Index out of bounds in dimension ^%d!' % d) offset += s * strides[d]; elif s is np.newaxis or s is None: d -= 1 else: raise IndexError('Invalid indexing object %r' % s); return offset;
[docs] def sliced_strides(slicing, strides): """Returns the strides of the slicing of a buffer with given strides if possible. Arguments --------- slicing : object Slice specification. strides : tuple Strides of the original array. Returns ------- strides : tuple Strides into the sliced array. """ slicing = simplify_slicing(slicing, len(strides)); sliced = []; d = -1; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): pass; elif isinstance(s, slice): sliced.append(strides[d] * (s.step or 1)); elif isinstance(s, np.ndarray): raise ValueError('Fancy slicing does not result in valid strides!') elif s is np.newaxis or s is None: sliced.append(0); d -= 1; else: raise IndexError('Invalid indexing object %r' % s); return tuple(sliced);
[docs] def sliced_start(slicing, shape, allow_index_arrays = allow_index_arrays): """Returns the starting position of the slicing in the original source. Arguments --------- slicing : object Slice specification. shape : tuple Shape of the array. Returns ------- start : tuple of int Start position of the slicing in the original source. """ slicing = unpack_slicing(slicing, len(shape)); start = []; index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; d = -1; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): if s < 0: s = shape[d] + s; if s < 0: raise IndexError('Index out of bounds in dimension %d!' % s) start.append(s); elif isinstance(s, slice): s = (s.start or 0); if s < 0: s = shape[d] + s; s = 0 if s < 0 else s; start.append(s); elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d) start = np.where(s)[0]; if len(start) == 0: raise IndexError('There is not True value in boolean array slicing in dimension %d!' % d) start.append(start[0]); elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) if len(s) == 0: raise IndexError('There is no index in array slicing in dimension %d!' % d) s = s[0]; if s < 0: s = shape[d] + s; if s < 0 : raise IndexError('Index out of bounds in dimension %d' % d) start.append(s); elif s is np.newaxis or s is None: d -= 1 else: raise IndexError('Invalid indexing object %r' % s); return tuple(start);
[docs] def sliced_slicing(slicing_second, slicing_first, shape, allow_index_arrays = allow_index_arrays): """Returns a slicing of a slicing if possible. Arguments --------- slicing_second : object Slice specification followed by first slicing. slicing_first : object First slicing. shape : tuple of ints Shape of the original source to be sliced twice. Returns ------- slicing : object The reduced slicing. """ shape1 = shape; slicing1 = simplify_slicing(slicing_first, len(shape1)); shape2 = sliced_shape(slicing1, shape1); slicing2 = simplify_slicing(slicing_second, len(shape2)); #print slicing1, slicing2, shape1, shape2 d1 = -1; d2 = -1; slicing = []; index_array1 = 0; bool_array1 = 0; index_array2 = 0; bool_array2 = 0; max_arrays = 1 if allow_index_arrays else 0; for s1 in slicing1: s1 = _standard_slice(s1); if isinstance(s1, int): d1 += 1; if s1 > shape1[d1] or -s1 > shape1[d1]: raise ValueError('Integer index %d out of range in dimension %d!' % (s1, d1)); slicing.append(s1); elif isinstance(s1, slice): d1 += 1; start1,stop1,step1 = s1.indices(shape1[d1]); d2 += 1; s2 = _standard_slice(slicing2[d2]); while s2 is np.newaxis or s2 is None: d2 += 1; s2 = _standard_slice(slicing2[d2]); slicing.append(np.newaxis); if isinstance(s2, int): if s2 < 0: s = stop1 + step1 * s2; else: s = start1 + step1 * s2; if s < start1 or s > stop1 or s > shape1[d1]: raise ValueError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); slicing.append(s); elif isinstance(s2, slice): start2, stop2, step2 = s2.indices(shape2[d2]); start = start1 + start2; stop = start1 + stop2; stop = min(stop1, stop); step = step1 * step2; start = None if start == 0 else start; stop = None if stop == shape1[d1] else stop; step = None if step == 1 else step; slicing.append(slice(start, stop, step)); elif isinstance(s2, np.ndarray) and s2.dtype == bool: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2); bool_array2 += 1; if bool_array2 > max_arrays: raise IndexError('Second boolean array slicing in dimension %d not supported!' % d2); if len(s2) != shape2[d2]: raise IndexError('Second boolean array slicing with shape %d is not of shape %d in dimension %d!' % (len(s2), shape2[d2], d2)); slicing.append(np.arange(start1, stop1, step1)[s2]); elif isinstance(s2, np.ndarray) and s2.dtype == int: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2) index_array2 += 1; if index_array2 > max_arrays: raise IndexError('Second index array slicing in dimension %d not supported!' % d2) try: slicing.append(np.arange(start1, stop1, step1)[s2]); except: raise IndexError('Index out of range in second array slicing in dimension %d!' % d2) else: raise IndexError('The index at dimension %d in second slicing is invalid!' % d2) elif isinstance(s1, np.ndarray) and s1.dtype == bool: d1 += 1; if s1.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d1); bool_array1 += 1; if bool_array1 > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d1) if len(s1) != shape1[d1]: raise IndexError('Boolean array slicing with shape %d is not of shape %d in dimension %d!' % (len(s1), shape1[d1], d1)); s1 = np.where(s1)[0]; d2 += 1; s2 = _standard_slice(slicing2[d2]); while s2 is np.newaxis or s2 is None: d2 += 1; s2 = _standard_slice(slicing2[d2]); slicing.append(np.newaxis); if isinstance(s2, int): try: s = s1[s2]; except: raise IndexError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); if s > shape1[d1] or -s > shape1[d1]: raise ValueError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); slicing.append(s); elif isinstance(s2, slice): try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) elif isinstance(s2, np.ndarray) and s2.dtype == bool: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2); bool_array2 += 1; if bool_array2 > max_arrays: raise IndexError('Second boolean array slicing in dimension %d not supported!' % d2); if len(s2) != shape2[d2]: raise IndexError('Second boolean array slicing with shape %d is not an array with shape %d in dimension %d!' % (len(s2), shape2[d2], d2)); s1 = np.where(s1)[0]; try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) elif isinstance(s2, np.ndarray) and s2.dtype == int: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2) index_array2 += 1; if index_array2 > max_arrays: raise IndexError('Second index array slicing in dimension %d not supported!' % d2) if len(s2) == 0: raise IndexError('There is no index in second array slicing in dimension %d!' % d2) try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) else: raise IndexError('The index at dimension %d in second slicing is invalid!' % d2) elif isinstance(s1, np.ndarray) and s1.dtype == int: d1 += 1; if s1.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d1); index_array1 += 1; if index_array1 > max_arrays: raise IndexError('Integer array slicing in dimension %d not supported!' % d1) if len(s1) == 0: raise IndexError('There is no index in array slicing in dimension %d!' % d1) d2 += 1; s2 = _standard_slice(slicing2[d2]); while s2 is np.newaxis or s2 is None: d2 += 1; s2 = _standard_slice(slicing2[d2]); slicing.append(np.newaxis); if isinstance(s2, int): try: s = s1[s2]; except: raise IndexError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); if s > shape1[d1] or -s > shape1[d1]: raise ValueError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); slicing.append(s); elif isinstance(s2, slice): try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) elif isinstance(s2, np.ndarray) and s2.dtype == bool: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2); bool_array2 += 1; if bool_array2 > max_arrays: raise IndexError('Second boolean array slicing in dimension %d not supported!' % d2); if len(s2) != shape2[d2]: raise IndexError('Second boolean array slicing with shape %d is not an array with shape %d in dimension %d!' % (len(s2), shape2[d2], d2)); try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) elif isinstance(s2, np.ndarray) and s2.dtype == int: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2) index_array2 += 1; if index_array2 > max_arrays: raise IndexError('Second index array slicing in dimension %d not supported!' % d2) if len(s2) == 0: raise IndexError('There is no index in second array slicing in dimension %d!' % d2) try: slicing.append(s1[s2]); except: raise IndexError('Index out of range in second slicing in dimension %d!' % d2) else: raise IndexError('The index at dimension %d in second slicing is invalid!' % d2) elif s1 is np.newaxis or s2 is None: d2 += 1; s2 = _standard_slice(slicing2[d2]); while s2 is np.newaxis or s2 is None: d2 += 1; s2 = _standard_slice(slicing2[d2]); slicing.append(np.newaxis); assert shape2[d2] == 1 if isinstance(s2, int): if s2 not in [0,-1]: raise IndexError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); elif isinstance(s2, slice): start, stop, step = s2.indices(1); if start == 1 or stop == 0: raise IndexError('Empty slice of a new axis cannot be reduced in dimension %d!' % d2); else: slicing.append(np.newaxis); elif isinstance(s2, np.ndarray) and s2.dtype == bool: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2); bool_array2 += 1; if bool_array2 > max_arrays: raise IndexError('Second boolean array slicing in dimension %d not supported!' % d2); if len(s2) != shape2[d2]: raise IndexError('Second boolean array slicing with shape %d is not of shape %d in dimension %d!' % (len(s2), shape2[d2], d2)); if s2[0] == False: raise IndexError('Empty slice of a new axis cannot be reduced in dimension %d!' % d2); else: slicing.append(np.newaxis); elif isinstance(s2, np.ndarray) and s2.dtype == int: if s2.ndim != 1: raise IndexError('Fancy second slicing in dimension %d not supported!' % d2) index_array2 += 1; if index_array2 > max_arrays: raise IndexError('Second index array slicing in dimension %d not supported!' % d2) if len(s2) != shape2[d2]: raise IndexError('Second boolean array slicing with shape %d is not of shape %d in dimension %d!' % (len(s2), shape2[d2], d2)); if s2[0] not in [0,-1]: raise IndexError('Index %d in second slicing out of range in dimension %d!' % (s, d2)); slicing.append(np.newaxis); else: raise IndexError('The index at dimension %d in second slicing is invalid!' % d2) else: raise IndexError('The index at dimension %d in first slicing is invalid!' % d1) d2 += 1; while d2 < len(slicing2): if not (slicing2[d2] is np.newaxis or slicing2[d2] is None): raise IndexError('The index at dimension %d in second slicing is invalid!' % d2); else: slicing.append(np.newaxis); return tuple(slicing);
[docs] def sliced_reduction(slicing, ndim, allow_index_arrays = allow_index_arrays): """Returns a slicing that slices a list retaining only full dimensions in the slice. Arguments --------- slicing : object The slice specification. ndim : int The dinension of the source. Returns ------- slicing : object Slice specification that reduces a list of length ndim to the new dimensions of the slice. """ slicing = unpack_slicing(slicing, ndim); reduction = []; index_array = 0; bool_array = 0; max_arrays = 1 if allow_index_arrays else 0; d = -1; for s in slicing: d += 1; s = _standard_slice(s); if isinstance(s, int): continue; elif isinstance(s, slice): reduction.append(d); elif isinstance(s, np.ndarray) and s.dtype == bool: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d); bool_array += 1; if bool_array > max_arrays: raise IndexError('Boolean array slicing in dimension %d not supported!' % d) reduction.append(d); elif isinstance(s, np.ndarray) and s.dtype == int: if s.ndim != 1: raise IndexError('Fancy slicing in dimension %d not supported!' % d) index_array += 1; if index_array > max_arrays: raise IndexError('Index array slicing in dimension %d not supported!' % d) if len(s) == 0: raise IndexError('There is no index in array slicing in dimension %d!' % d) reduction.append(d); elif s is np.newaxis or s is None: d -= 1 else: raise IndexError('Invalid indexing object %r' % s); return reduction;
############################################################################### ### Helpers ############################################################################### def _standard_slice(s): #, top_level=False): #if top_level and isinstance(s, list): # return [_standard_slice(si, top_level=False) for si in s]; if s is Ellipsis: return s; if isinstance(s, numbers.Integral): return int(s); if isinstance(s, slice): return s; if isinstance(s, (list, tuple, np.ndarray)): s = np.asarray(s); else: try: iter(s); except: return s; else: s = np.asarray(s); if not s.dtype in [bool, int]: s = np.asarray(s, dtype = int); return s; def _slicing_to_str(slicing, ndim): slicing = unpack_slicing(slicing, ndim); info = '('; for s in slicing: s = _standard_slice(s); if s is Ellipsis: info += ':' elif isinstance(s, slice): if s.start is None and s.stop is None and s.step is None: info += ':' else: for r in [s.start, s.stop, s.step]: if r is not None: info += '%d' % r; info += ':' if s.step is None: info = info[:-1]; info = info[:-1]; else: info += '%r' % s; info += ',' info = info[:-1] + ')'; return info; ############################################################################### ### Tests ############################################################################### def _test(): import numpy as np; # analysis:ignore import ClearMap.IO.Slice as slc; from importlib import reload reload(slc) s1 = (slice(1,4), [1,2,3,4,5], None, Ellipsis); ss = slc.simplify_slicing(s1, ndim = 5); print(ss) shape = (7,6,2,3,5); d1 = slc.sliced_ndim(s1, 5, allow_index_arrays = True) shape1 = slc.sliced_shape(s1, shape, allow_index_arrays=True) print(d1, shape1) x = np.random.rand(*shape); x1 = x[s1]; x1.shape == shape1 s2 = (slice(None, None, 2), slice(3,4), slice(None), 1, [0,2,1]); s12 = slc.sliced_slicing(s2, s1, shape, allow_index_arrays=True); np.all(x[s12] == x[s1][s2]) slc.is_view(s1) s1s = slc.simplify_slicing(s1) slc.is_view(s1s) s2 = (slice(None, None, 2), slice(3,4), slice(None), 1, slice(0,2)); y = x[s1s][s2] y.base is x s12 = slc.sliced_slicing(s2, s1s, shape); slc.is_view(s12) x[s12].base is x x = slc.src.VirtualSource(shape = (5,10,15), dtype = float, order = 'F', location = '/home/test.src') s = slc.Slice(source = x, slicing = (Ellipsis, 1)); print(s) print(s.source) print(s.unpacked_slicing) reload(slc) shape = (50,100,200) s1 = (slice(None), slice(None), slice(0, 38)); s2 = (slice(None), slice(None), slice(None, -10)); slc.sliced_slicing(s2,s1,shape)