Base class containing the implementation details that are independent of the item type. More...
#include <tpie/compressed/stream.h>
Inherited by tpie::file_stream< element_type >, tpie::file_stream< item_type >, tpie::file_stream< tpie::blocks::block_handle >, tpie::file_stream< tpie::stream_position >, and tpie::file_stream< T >.
Classes | |
struct | seek_state |
Public Types | |
typedef std::shared_ptr < compressor_buffer > | buffer_t |
Public Member Functions | |
bool | is_readable () const throw () |
bool | is_writable () const throw () |
memory_size_type | block_items () const |
memory_size_type | block_size () const |
template<typename TT > | |
void | read_user_data (TT &data) |
memory_size_type | read_user_data (void *data, memory_size_type count) |
template<typename TT > | |
void | write_user_data (const TT &data) |
void | write_user_data (const void *data, memory_size_type count) |
memory_size_type | user_data_size () const |
memory_size_type | max_user_data_size () const |
const std::string & | path () const |
void | open (const std::string &path, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening a named stream. More... | |
void | open (memory_size_type userDataSize, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening an unnamed temporary stream. More... | |
void | open (temp_file &file, access_type accessType, memory_size_type userDataSize=0, cache_hint cacheHint=access_sequential, compression_flags compressionFlags=compression_none) |
Deprecated interface for opening a temporary stream. More... | |
void | open (const std::string &path, compression_flags compressionFlags) |
Deprecated interface for opening a named stream. More... | |
void | open (compression_flags compressionFlags) |
Deprecated interface for opening an unnamed temporary stream. More... | |
void | open (temp_file &file, compression_flags compressionFlags) |
Deprecated interface for opening a temporary stream. More... | |
void | open (const std::string &path, open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and possibly create a stream. More... | |
void | open (open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and create an unnamed temporary stream. More... | |
void | open (temp_file &file, open::type openFlags=open::defaults, memory_size_type userDataSize=0) |
Open and possibly create a temporary stream. More... | |
void | close () |
bool | is_open () const |
stream_size_type | size () const |
stream_size_type | file_size () const |
stream_size_type | offset () const |
Protected Member Functions | |
compressed_stream_base (memory_size_type itemSize, double blockFactor) | |
virtual void | flush_block (compressor_thread_lock &)=0 |
virtual void | post_open ()=0 |
void | open_inner (const std::string &path, open::type openFlags, memory_size_type userDataSize) |
compressor_thread & | compressor () |
void | finish_requests (compressor_thread_lock &l) |
stream_size_type | last_block_read_offset (compressor_thread_lock &l) |
Blocks to take the compressor lock. More... | |
stream_size_type | current_file_size (compressor_thread_lock &l) |
Blocks to take the compressor lock. More... | |
bool | use_compression () |
void | uncache_read_writes () |
Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly. More... | |
Protected Attributes | |
bool | m_bufferDirty |
Whether the current block must be written out to disk before being ejected. More... | |
memory_size_type | m_blockItems |
Number of items in a logical block. More... | |
memory_size_type | m_blockSize |
Size (in bytes) of a logical (uncompressed) block. More... | |
bool | m_canRead |
Whether we are open for reading. More... | |
bool | m_canWrite |
Whether we are open for writing. More... | |
bool | m_open |
Whether we are open. More... | |
memory_size_type | m_itemSize |
Size of a single item. More... | |
memory_size_type | m_cachedReads |
Number of cheap, unchecked reads we can do next. More... | |
memory_size_type | m_cachedWrites |
Number of cheap, unchecked writes we can do next. More... | |
tpie::unique_ptr< temp_file > | m_ownedTempFile |
The anonymous temporary file we have opened (when appropriate). More... | |
temp_file * | m_tempFile |
The temporary file we have opened (when appropriate). More... | |
file_accessor::byte_stream_accessor < default_raw_file_accessor > | m_byteStreamAccessor |
File accessor. More... | |
stream_size_type | m_size |
Number of logical items in the stream. More... | |
stream_buffers | m_buffers |
Buffer manager for this entire stream. More... | |
buffer_t | m_buffer |
Buffer holding the items of the block currently being read/written. More... | |
stream_size_type | m_streamBlocks |
The number of blocks written to the file. More... | |
stream_size_type | m_lastBlockReadOffset |
When use_compression() is true: Read offset of the last block in the stream. More... | |
stream_size_type | m_currentFileSize |
compressor_response | m_response |
Response from compressor thread; protected by compressor thread mutex. More... | |
bool | m_updateReadOffsetFromWrite = false |
When use_compression() is true: Indicates whether m_response is the response to a write request. More... | |
stream_size_type | m_lastWriteBlockNumber |
seek_state::type | m_seekState |
stream_size_type | m_readOffset |
Position relating to the currently loaded buffer. More... | |
stream_size_type | m_offset |
Offset of next item to read/write, relative to beginning of stream. More... | |
stream_position | m_nextPosition |
If seekState is position , seek to this position before reading/writing. More... | |
stream_size_type | m_nextReadOffset |
Base class containing the implementation details that are independent of the item type.
|
protected |
Blocks to take the compressor lock.
Precondition: use_compression()
TODO: Should probably investigate when this reports a useful value.
Referenced by tpie::file_stream< tpie::stream_position >::get_position().
|
protected |
Blocks to take the compressor lock.
Precondition: use_compression()
TODO: Should probably investigate when this reports a useful value.
Referenced by tpie::file_stream< tpie::stream_position >::get_position().
|
inline |
Deprecated interface for opening a named stream.
If compressionFlags is compression_none and the file does not already exist, no compression will be used when writing. If compressionFlags is compression_normal and the file does not already exist, compression will be used when writing. If the file already exists, the compression flags of the existing file are used instead.
Definition at line 204 of file stream.h.
Referenced by tpie::pipelining::bits::named_output_t< T >::begin(), open(), and tpie::pipelining::bits::named_pull_input_t< T >::propagate().
|
inline |
Deprecated interface for opening an unnamed temporary stream.
Definition at line 216 of file stream.h.
References tpie::access_read_write, and open().
|
inline |
Deprecated interface for opening a temporary stream.
Definition at line 225 of file stream.h.
References open().
|
inline |
Deprecated interface for opening a named stream.
Definition at line 236 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and open().
|
inline |
Deprecated interface for opening an unnamed temporary stream.
Definition at line 244 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and open().
|
inline |
Deprecated interface for opening a temporary stream.
Definition at line 252 of file stream.h.
References tpie::access_read_write, tpie::access_sequential, and open().
void tpie::compressed_stream_base::open | ( | const std::string & | path, |
open::type | openFlags = open::defaults , |
||
memory_size_type | userDataSize = 0 |
||
) |
Open and possibly create a stream.
The stream is created if it does not exist and opened for reading and writing, but this can be changed with open::read_only or open::write_only; see below.
The flags supplied to openFlags should be a combination of the following from open::type
, OR'ed together:
open::read_only Open for reading only, and fail if the stream does not exist.
open::write_only Open for writing only, and truncate the stream if it exists.
open::access_normal By default, POSIX_FADV_SEQUENTIAL is passed to the open syscall to indicate that the OS should optimize for sequential access; this flag disables that flag.
open::access_random Pass POSIX_FADV_RANDOM to the open syscall to make the OS optimize for random access.
open::compression_normal Create the stream in compression mode if it does not already exist, and compress written blocks according to available resources (for instance CPU time and memory).
open::compression_all Create the stream in compression mode if it does not already exist, and compress all written blocks using the preferred compression scheme, which can be set using tpie::the_compressor_thread().set_preferred_compression().
path | The path to the file to open |
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
void tpie::compressed_stream_base::open | ( | open::type | openFlags = open::defaults , |
memory_size_type | userDataSize = 0 |
||
) |
Open and create an unnamed temporary stream.
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
void tpie::compressed_stream_base::open | ( | temp_file & | file, |
open::type | openFlags = open::defaults , |
||
memory_size_type | userDataSize = 0 |
||
) |
Open and possibly create a temporary stream.
file | The temporary file to open |
openFlags | A bit-wise combination of the flags; see above. |
userDataSize | Required user data capacity in stream header. |
|
inlineprotected |
Reset cheap read/write counts to zero so that the next read/write operation will check stream state properly.
Definition at line 345 of file stream.h.
References m_cachedReads, and m_cachedWrites.
Referenced by tpie::file_stream< tpie::stream_position >::seek(), tpie::file_stream< tpie::stream_position >::set_position(), and tpie::file_stream< tpie::stream_position >::truncate().
|
protected |
Number of items in a logical block.
Definition at line 380 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().
|
protected |
|
protected |
Buffer holding the items of the block currently being read/written.
Definition at line 407 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protected |
Whether the current block must be written out to disk before being ejected.
Invariants: If m_bufferDirty is true and use_compression() is true, block_number() is either m_streamBlocks or m_streamBlocks - 1. If block_number() is m_streamBlocks, m_bufferDirty is true.
Definition at line 378 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::peek().
|
protected |
|
protected |
File accessor.
Definition at line 401 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe().
|
protected |
Number of cheap, unchecked reads we can do next.
Definition at line 392 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::can_read(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), and uncache_read_writes().
|
protected |
Number of cheap, unchecked writes we can do next.
Definition at line 394 of file stream.h.
Referenced by uncache_read_writes().
|
protected |
|
protected |
|
protected |
|
protected |
When use_compression() is true: Read offset of the last block in the stream.
Necessary to support seeking to the end.
Definition at line 416 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe().
|
protected |
If seekState is position
, seek to this position before reading/writing.
Definition at line 456 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protected |
Offset of next item to read/write, relative to beginning of stream.
Invariants:
block_number() in [0, m_streamBlocks] offset in [0, size] block_item_index() in [0, m_blockSize) offset == block_number() * m_blockItems + block_item_index()
block_item_index() <= offset.
If block_number() == m_streamBlocks, we are in a block that has not yet been written to disk.
Definition at line 453 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::read(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protected |
Whether we are open.
Definition at line 388 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::can_read(), and tpie::file_stream< tpie::stream_position >::can_read_back().
|
protected |
|
protected |
Position relating to the currently loaded buffer.
readOffset is only valid during reading. Invariants:
If use_compression() == false, readOffset == 0. If offset == 0, then readOffset == block_item_index() == block_number() == 0.
Definition at line 438 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().
|
protected |
|
protected |
Number of logical items in the stream.
Definition at line 403 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::peek(), and tpie::file_stream< tpie::stream_position >::truncate().
|
protected |
The number of blocks written to the file.
We must always have (m_streamBlocks+1) * m_blockItems <= m_size.
Definition at line 411 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::describe(), tpie::file_stream< tpie::stream_position >::get_position(), and tpie::file_stream< tpie::stream_position >::seek().
|
protected |
The temporary file we have opened (when appropriate).
When m_ownedTempFile.get() != 0, m_tempFile == m_ownedTempFile.get().
Definition at line 399 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::truncate().
|
protected |
When use_compression() is true: Indicates whether m_response is the response to a write request.
Used for knowing where to read next in read/read_back.
Definition at line 426 of file stream.h.
Referenced by tpie::file_stream< tpie::stream_position >::get_position(), tpie::file_stream< tpie::stream_position >::peek(), tpie::file_stream< tpie::stream_position >::seek(), and tpie::file_stream< tpie::stream_position >::set_position().