Compressed stream. More...
#include <tpie/compressed/stream.h>
Inherits tpie::compressed_stream_base.
Public Types | |
typedef T | item_type |
typedef file_stream_base::offset_type | offset_type |
typedef std::shared_ptr < compressor_buffer > | buffer_t |
Public Member Functions | |
file_stream (double blockFactor=1.0) | |
void | describe (std::ostream &out) |
For debugging: Describe the internal stream state in a string. More... | |
std::string | describe () |
For debugging: Describe the internal stream state in a string. More... | |
virtual void | post_open () override |
void | seek (stream_offset_type offset, offset_type whence=beginning) |
Precondition: is_open() Precondition: offset == 0. More... | |
void | truncate (stream_size_type offset) |
Truncate to given size. More... | |
void | truncate (const stream_position &pos) |
Truncate to given stream position. More... | |
stream_position | get_position () |
Store the current stream position such that it may be found later on. More... | |
void | set_position (const stream_position &pos) |
Seek to a position that was previously recalled with get_position . More... | |
const T & | read () |
Reads next item from stream if can_read() == true. More... | |
const T & | peek () |
Peeks next item from stream if can_read() == true. More... | |
void | skip () |
void | skip_back () |
template<typename IT > | |
void | read (IT const a, IT const b) |
Precondition: is_open(). More... | |
bool | can_read () |
Check if the next call to read() will succeed or not. More... | |
bool | can_read_back () |
Check if the next call to read_back() will succeed or not. More... | |
const T & | read_back () |
void | write (const T &item) |
template<typename IT > | |
void | write (IT const a, IT const b) |
bool | is_readable () const throw () |
bool | is_writable () const throw () |
memory_size_type | block_size () const |
memory_size_type | block_items () const |
template<typename TT > | |
void | read_user_data (TT &data) |
memory_size_type | read_user_data (void *data, memory_size_type count) |
template<typename TT > | |
void | write_user_data (const TT &data) |
void | write_user_data (const void *data, memory_size_type count) |
memory_size_type | user_data_size () const |
memory_size_type | max_user_data_size () const |
const std::string & | path () const |
void | open (const std::string &path, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening a named stream. More... | |
void | open (memory_size_type userDataSize, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening an unnamed temporary stream. More... | |
void | open (temp_file &file, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening a temporary stream. More... | |
void | open (const std::string &path, compression_flags compressionFlags) |
Deprecated interface for opening a named stream. More... | |
void | open (compression_flags compressionFlags) |
Deprecated interface for opening an unnamed temporary stream. More... | |
void | open (temp_file &file, compression_flags compressionFlags) |
Deprecated interface for opening a temporary stream. More... | |
void | open (const std::string &path, open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and possibly create a stream. More... | |
void | open (open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and create an unnamed temporary stream. More... | |
void | open (temp_file &file, open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and possibly create a temporary stream. More... | |
void | close () |
bool | is_open () const |
stream_size_type | size () const |
stream_size_type | file_size () const |
stream_size_type | offset () const |
Static Public Attributes | |
static const file_stream_base::offset_type | beginning = file_stream_base::beginning |
static const file_stream_base::offset_type | end = file_stream_base::end |
static const file_stream_base::offset_type | current = file_stream_base::current |
Protected Member Functions | |
void | open_inner (const std::string &path, open::type openFlags, memory_size_type userDataSize) |
compressor_thread & | compressor () |
void | finish_requests (compressor_thread_lock &l) |
stream_size_type | last_block_read_offset (compressor_thread_lock &l) |
Blocks to take the compressor lock. More... | |
stream_size_type | current_file_size (compressor_thread_lock &l) |
Blocks to take the compressor lock. More... | |
bool | use_compression () |
void | uncache_read_writes () |
Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly. More... | |
Protected Attributes | |
bool | m_bufferDirty |
Whether the current block must be written out to disk before being ejected. More... | |
memory_size_type | m_blockItems |
Number of items in a logical block. More... | |
memory_size_type | m_blockSize |
Size (in bytes) of a logical (uncompressed) block. More... | |
bool | m_canRead |
Whether we are open for reading. More... | |
bool | m_canWrite |
Whether we are open for writing. More... | |
bool | m_open |
Whether we are open. More... | |
memory_size_type | m_itemSize |
Size of a single item. More... | |
memory_size_type | m_cachedReads |
Number of cheap, unchecked reads we can do next. More... | |
memory_size_type | m_cachedWrites |
Number of cheap, unchecked writes we can do next. More... | |
tpie::unique_ptr< temp_file > | m_ownedTempFile |
The anonymous temporary file we have opened (when appropriate). More... | |
temp_file * | m_tempFile |
The temporary file we have opened (when appropriate). More... | |
file_accessor::byte_stream_accessor < default_raw_file_accessor > | m_byteStreamAccessor |
File accessor. More... | |
stream_size_type | m_size |
Number of logical items in the stream. More... | |
stream_buffers | m_buffers |
Buffer manager for this entire stream. More... | |
buffer_t | m_buffer |
Buffer holding the items of the block currently being read/written. More... | |
stream_size_type | m_streamBlocks |
The number of blocks written to the file. More... | |
stream_size_type | m_lastBlockReadOffset |
When use_compression() is true: Read offset of the last block in the stream. More... | |
stream_size_type | m_currentFileSize |
compressor_response | m_response |
Response from compressor thread; protected by compressor thread mutex. More... | |
bool | m_updateReadOffsetFromWrite = false |
When use_compression() is true: Indicates whether m_response is the response to a write request. More... | |
stream_size_type | m_lastWriteBlockNumber |
seek_state::type | m_seekState |
stream_size_type | m_readOffset |
Position relating to the currently loaded buffer. More... | |
stream_size_type | m_offset |
Offset of next item to read/write, relative to beginning of stream. More... | |
stream_position | m_nextPosition |
If seekState is position , seek to this position before reading/writing. More... | |
stream_size_type | m_nextReadOffset |
Compressed stream.
We assume that T
is trivially copyable and that its copy constructor and assignment operator never throws.
As a rule of thumb, when a tpie::stream_exception
is thrown from a method, the stream is left in the state it was in prior to the method call. When a tpie::exception
is thrown, the stream may have changed. In particular, the stream may have been closed, and it is up to the caller (if the exception is caught) to ensure that the stream is reopened as necessary.
Several methods claim the nothrow
guarantee even though the implementation has throw
statements. In this case, there are two reasons an exception may be thrown: A tpie::exception
is thrown if some invariant in the stream has been violated, and this is a bug we must fix in the compressed stream. A tpie::stream_exception
is thrown if the user has violated a precondition (for instance by passing an invalid parameter).
Definition at line 46 of file predeclare.h.
|
inline |
Check if the next call to read() will succeed or not.
Definition at line 1010 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::pipelining::bits::input_t< dest_t >::go(), tpie::pipelining::bits::buffer_output_t< dest_t >::go(), and tpie::ami::merge_sorted_runs().
|
inline |
|
protectedinherited |
Blocks to take the compressor lock.
Precondition: use_compression()
TODO: Should probably investigate when this reports a useful value.
Referenced by tpie::file_stream< tpie::stream_position >::get_position().
|
inline |
For debugging: Describe the internal stream state in a string.
Definition at line 556 of file stream.h.
|
inline |
For debugging: Describe the internal stream state in a string.
Definition at line 613 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe().
|
inline |
Store the current stream position such that it may be found later on.
The stream_position object is violated if the stream is eventually truncated to before the current position.
The stream_position objects are plain old data, so they may themselves be written to streams.
Blocks to take the compressor lock.
Definition at line 841 of file stream.h.
|
protectedinherited |
Blocks to take the compressor lock.
Precondition: use_compression()
TODO: Should probably investigate when this reports a useful value.
Referenced by tpie::file_stream< tpie::stream_position >::get_position().
|
inlineinherited |
Deprecated interface for opening a named stream.
If compressionFlags is compression_none and the file does not already exist, no compression will be used when writing. If compressionFlags is compression_normal and the file does not already exist, compression will be used when writing. If the file already exists, the compression flags of the existing file are used instead.
Definition at line 204 of file stream.h.
Referenced by tpie::pipelining::bits::named_output_t< T >::begin(), tpie::compressed_stream_base::open(), and tpie::pipelining::bits::named_pull_input_t< T >::propagate().
|
inlineinherited |
Deprecated interface for opening an unnamed temporary stream.
Definition at line 216 of file stream.h.
References tpie::access_read_write, and tpie::compressed_stream_base::open().
|
inlineinherited |
Deprecated interface for opening a temporary stream.
Definition at line 225 of file stream.h.
References tpie::compressed_stream_base::open().
|
inlineinherited |
Deprecated interface for opening a named stream.
Definition at line 236 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and tpie::compressed_stream_base::open().
|
inlineinherited |
Deprecated interface for opening an unnamed temporary stream.
Definition at line 244 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and tpie::compressed_stream_base::open().
|
inlineinherited |
Deprecated interface for opening a temporary stream.
Definition at line 252 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and tpie::compressed_stream_base::open().
|
inherited |
Open and possibly create a stream.
The stream is created if it does not exist and opened for reading and writing, but this can be changed with open::read_only or open::write_only; see below.
The flags supplied to openFlags should be a combination of the following from open::type
, OR'ed together:
open::read_only Open for reading only, and fail if the stream does not exist.
open::write_only Open for writing only, and truncate the stream if it exists.
open::access_normal By default, POSIX_FADV_SEQUENTIAL is passed to the open syscall to indicate that the OS should optimize for sequential access; this flag disables that flag.
open::access_random Pass POSIX_FADV_RANDOM to the open syscall to make the OS optimize for random access.
open::compression_normal Create the stream in compression mode if it does not already exist, and compress written blocks according to available resources (for instance CPU time and memory).
open::compression_all Create the stream in compression mode if it does not already exist, and compress all written blocks using the preferred compression scheme, which can be set using tpie::the_compressor_thread().set_preferred_compression().
path | The path to the file to open |
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
|
inherited |
Open and create an unnamed temporary stream.
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
|
inherited |
Open and possibly create a temporary stream.
file | The temporary file to open |
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
|
inline |
Peeks next item from stream if can_read() == true.
If can_read() == false, throws an end_of_stream_exception.
Blocks to take the compressor lock.
If a stream_exception is thrown, the stream is left in the state it was in before the call to peek().
Definition at line 970 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::read().
|
inline |
Reads next item from stream if can_read() == true.
If can_read() == false, throws an end_of_stream_exception.
Blocks to take the compressor lock.
If a stream_exception is thrown, the stream is left in the state it was in before the call to read().
Definition at line 947 of file stream.h.
Referenced by tpie::pipelining::bits::input_t< dest_t >::go(), tpie::pipelining::bits::buffer_output_t< dest_t >::go(), tpie::ami::merge_sorted_runs(), tpie::file_stream< tpie::stream_position >::read(), tpie::sort_manager< T, I, M >::sort(), and tpie::ami::Internal_Sorter_Obj< T, Compare >::sort().
|
inline |
Precondition: is_open().
Reads min(b-a, size()-offset()) items into the range [a, b). If less than b-a items are read, throws an end_of_stream_exception.
Definition at line 1003 of file stream.h.
|
inline |
Precondition: is_open() Precondition: offset == 0.
Definition at line 627 of file stream.h.
Referenced by tpie::pipelining::bits::buffer_output_t< dest_t >::go(), tpie::pipelining::bits::input_t< dest_t >::propagate(), tpie::pipelining::bits::pull_input_t< T >::propagate(), tpie::pipelining::bits::pull_reverse_input_t< T >::propagate(), tpie::file_stream< tpie::stream_position >::set_position(), tpie::sort_manager< T, I, M >::sort(), and tpie::ami::Internal_Sorter_Obj< T, Compare >::sort().
|
inline |
Seek to a position that was previously recalled with get_position
.
Definition at line 893 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::seek().
|
inline |
Truncate to given size.
Precondition: compression is disabled or offset is size() or 0. Blocks to take the compressor lock.
Definition at line 698 of file stream.h.
Referenced by tpie::ami::Internal_Sorter_Obj< T, Compare >::sort().
|
inline |
Truncate to given stream position.
Definition at line 716 of file stream.h.
|
inlineprotectedinherited |
Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly.
Definition at line 345 of file stream.h.
References tpie::compressed_stream_base::m_cachedReads, and tpie::compressed_stream_base::m_cachedWrites.
Referenced by tpie::file_stream< tpie::stream_position >::seek(), tpie::file_stream< tpie::stream_position >::set_position(), and tpie::file_stream< tpie::stream_position >::truncate().
|
protectedinherited |
Number of items in a logical block.
Definition at line 380 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().
|
protectedinherited |
|
protectedinherited |
Buffer holding the items of the block currently being read/written.
Definition at line 407 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protectedinherited |
Whether the current block must be written out to disk before being ejected.
Invariants: If m_bufferDirty is true and use_compression() is true, block_number() is either m_streamBlocks or m_streamBlocks - 1. If block_number() is m_streamBlocks, m_bufferDirty is true.
Definition at line 378 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::peek().
|
protectedinherited |
|
protectedinherited |
File accessor.
Definition at line 401 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe().
|
protectedinherited |
Number of cheap, unchecked reads we can do next.
Definition at line 392 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::can_read(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), and tpie::compressed_stream_base::uncache_read_writes().
|
protectedinherited |
Number of cheap, unchecked writes we can do next.
Definition at line 394 of file stream.h.
Referenced by tpie::compressed_stream_base::uncache_read_writes().
|
protectedinherited |
|
protectedinherited |
|
protectedinherited |
|
protectedinherited |
When use_compression() is true: Read offset of the last block in the stream.
Necessary to support seeking to the end.
Definition at line 416 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe().
|
protectedinherited |
If seekState is position
, seek to this position before reading/writing.
Definition at line 456 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protectedinherited |
Offset of next item to read/write, relative to beginning of stream.
Invariants:
block_number() in [0, m_streamBlocks] offset in [0, size] block_item_index() in [0, m_blockSize) offset == block_number() * m_blockItems + block_item_index()
block_item_index() <= offset.
If block_number() == m_streamBlocks, we are in a block that has not yet been written to disk.
Definition at line 453 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protectedinherited |
Whether we are open.
Definition at line 388 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::can_read(), and tpie::file_stream< tpie::stream_position >::can_read_back().
|
protectedinherited |
|
protectedinherited |
Position relating to the currently loaded buffer.
readOffset is only valid during reading. Invariants:
If use_compression() == false, readOffset == 0. If offset == 0, then readOffset == block_item_index() == block_number() == 0.
Definition at line 438 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protectedinherited |
|
protectedinherited |
Number of logical items in the stream.
Definition at line 403 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::peek(), and tpie::file_stream< tpie::stream_position >::truncate().
|
protectedinherited |
The number of blocks written to the file.
We must always have (m_streamBlocks+1) * m_blockItems <= m_size.
Definition at line 411 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().
|
protectedinherited |
The temporary file we have opened (when appropriate).
When m_ownedTempFile.get() != 0, m_tempFile == m_ownedTempFile.get().
Definition at line 399 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::truncate().
|
protectedinherited |
When use_compression() is true: Indicates whether m_response is the response to a write request.
Used for knowing where to read next in read/read_back.
Definition at line 426 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().