FileList#

Module to handle sources distributed over a list of files.

File lists ar specified using a TagExpression.

class Source(expression=None, file_list=None, axes_order=None, shape=None, dtype=None, order=None, name=None)[source]#

Bases: VirtualSource

File list source.

Note

The full shape of the file list source is the shape of the expression and the shape of the data in each file, i.e. shape = file_list_shape + array_shape.

as_buffer()[source]#

as_real()[source]#

as_virtual()[source]#

Return virtual source without array data to pass in parallel processing.

Returns

sourceSource class: The source class without array data.

axes_to_tag_order()[source]#: Map from axes of this source to the tag list from the file expression.

tag_to_axes_order()[source]#: Map from the tag list from the file expression and the axes of this source.

property array#

property axes_file#: Source axes that constitute individual file dimensions in the full array.

property axes_list#: Source axes that constitute the dimensions of the file list in the full array.

property axes_order#

Optional ordering of the tag names.

Returns

axes_orderlist of str: The ordered axis names.

property dtype#

The data type of the source.

Returns

dtypedtype: The data type of the source.

property element_strides#

property expression#

The underlying expression of this file list.

Returns

expressionstr: The underlying expression of this source.

property file_list#

The underlying file list.

Returns

filelistlist: The underlying sources of this source.

property name#

The name of this source.

Returns

namestr: Name of this source.

property ndim_file#: Source dimension of the individual files.

property ndim_list#: Source dimensions of the file list.

property shape#

The shape of the source.

Returns

shapetuple: The shape of the source.

property shape_file#: Source shape of the individual files.

property shape_list#: Source shape of the file list.

class VirtualSource(expression=None, file_list=None, shape=None, dtype=None, order=None, axes_order=None, source=None, name=None)[source]#

Bases: VirtualSource

Virtual file list source.

as_buffer()[source]#

as_real()[source]#

as_virtual()[source]#

Return virtual source without array data to pass in parallel processing.

Returns

sourceSource class: The source class without array data.

property array#

property axes_order#

property dtype#

The data type of the source.

Returns

dtypedtype: The data type of the source.

property element_strides#

property expression#

The underlying expression of this file list.

Returns

expressionstr: The underlying expression of this source.

property file_list#

property name#

The name of this source.

Returns

namestr: Name of this source.

property shape#

The shape of the source.

Returns

shapetuple: The shape of the source.

convert(source, sink, processes=None, verbose=False)[source]#

Converts list of files to a sink in parallel

Arguments

sourceSource: File list source.
sinkSource: A sink to write the source to.

Returns

sinkSource: The sink the data was converted to.

create(location=None, shape=None, dtype=None, array=None, as_source=True)[source]#

dtype(expression=None, file_list=None)[source]#

Returns data type of the array stored in a file list.

Arguments

expressionstr: The regular epression for the file list.

Returns

dtypedtype: The data type of the file list.

is_file_list(expression, exists=False, tag_names=None, n_tags=-1, verbose=False)[source]#

Checks if the expression is a valid file list.

Arguments

expressionstr: The regular expression to check.
existsbool: If True, check if at least one file exists.
tag_nameslist of str or None: List of tag names expected to be present in the expression.
n_Tagsint or None: Number of tags to expect.
verbosebool: If True, print reason why the epxression does not represent the desired file list.

Returns

is_expressionbool: Returns True if the expression fullfills the desired criteria and at least one file matching the expression exists.

ndim(expression=None, file_list=None)[source]#

Calculates the dimension of the file list given by an expression.

Arguments

expressionstr: The expression for the file_list.
file_listlist: Optional file_list to speed up calculation.

Returns

ndimint: The dimension of the file list given by the expression.

order(expression=None, file_list=None)[source]#

Returns order of the array stored in a file list.

Arguments

expressionstr: The regular epression for the file list.

Returns

dtypedtype: The data type of the file list.

read(source, slicing=None, axes_order=None, **kwargs)[source]#

shape(expression=None, file_list=None, axes_order=None, axis=None)[source]#

Calculates the shape of the data in a file list.

Arguments

expressionstr or None: The regular epression for the file list.
file_listlist or None: List of files.
axisint or None: The shape along a specific axis. Can speed up the shape calculation.
axes_orderlist or None: The names of how to order the different tag names in the expression. If None, use ordering of the tags in the expression.

Returns

shapeint or tuple of ints: The shape of the array st ored in a file list.

shape_file(expression=None, file_list=None)[source]#

Calculates the shape of the data in a file list.

Arguments

expressionstr or None: The regular epression for the file list.
file_listlist or None: List of files.

Returns

shapeint or tuple of ints: The shape of the array st ored in a file list.

shape_list(expression=None, file_list=None, axes_order=None)[source]#

Calculates the shape of the data in a file list.

Arguments

expressionstr or None: The regular epression for the file list.
file_listlist or None: List of files.
axes_orderlist or None: The names of how to order the different tag names in the expression. If None, use ordering of the tags in the expression.

Returns

shapeint or tuple of ints: The shape of the array along the dimensions created by the file list.

write(sink, data, slicing=None, axes_order=None, processes=None, **kwargs)[source]#