/* This file is (c) 2008-2010 Konstantin Isakov * Part of GoldenDict. Licensed under GPLv3 or later, see the LICENSE file */ #include "chunkedstorage.hh" #include #include #ifdef _MSC_VER #include #else #include #endif namespace ChunkedStorage { enum { ChunkMaxSize = 65536 // Can't be more since it would overflow the address }; Writer::Writer( File::Class & f ): file( f ), chunkStarted( false ), bufferUsed( 0 ) { // Create a sratchpad at the beginning of file. We use it to write chunk // table if it would fit, in order to save some seek times. char zero[ 4096 ]; memset( zero, 0, sizeof( zero ) ); scratchPadOffset = file.tell(); scratchPadSize = sizeof( zero ); file.write( zero, sizeof( zero ) ); } uint32_t Writer::startNewBlock() { if ( bufferUsed >= ChunkMaxSize ) { // Need to flush first. saveCurrentChunk(); } chunkStarted = true; // The address is comprised of the offset within the chunk (in lower // 16 bits, always fits there since ChunkMaxSize-1 does) and the // number of the chunk, which is therefore limited to be 65535 max. return bufferUsed | ( (uint32_t)offsets.size() << 16 ); } void Writer::addToBlock( void const * data, size_t size ) { if ( !size ) return; if ( buffer.size() - bufferUsed < size ) buffer.resize( bufferUsed + size ); memcpy( &buffer.front() + bufferUsed, data, size ); bufferUsed += size; chunkStarted = false; } void Writer::saveCurrentChunk() { size_t maxCompressedSize = compressBound( bufferUsed ); if ( bufferCompressed.size() < maxCompressedSize ) bufferCompressed.resize( maxCompressedSize ); unsigned long compressedSize = bufferCompressed.size(); if ( compress( &bufferCompressed.front(), &compressedSize, &buffer.front(), bufferUsed ) != Z_OK ) throw exFailedToCompressChunk(); offsets.push_back( file.tell() ); file.write( (uint32_t) bufferUsed ); file.write( (uint32_t) compressedSize ); file.write( &bufferCompressed.front(), compressedSize ); bufferUsed = 0; chunkStarted = false; } uint32_t Writer::finish() { if ( bufferUsed || chunkStarted ) saveCurrentChunk(); bool useScratchPad = false; uint32_t savedOffset = 0; if ( scratchPadSize >= offsets.size() * sizeof( uint32_t ) + sizeof( uint32_t ) ) { useScratchPad = true; savedOffset = file.tell(); file.seek( scratchPadOffset ); } uint32_t offset = file.tell(); file.write( (uint32_t) offsets.size() ); if ( offsets.size() ) file.write( &offsets.front(), offsets.size() * sizeof( uint32_t ) ); if ( useScratchPad ) file.seek( savedOffset ); offsets.clear(); chunkStarted = false; return offset; } Reader::Reader( File::Class & f, uint32_t offset ): file( f ) { file.seek( offset ); offsets.resize( file.read< uint32_t >() ); file.read( &offsets.front(), offsets.size() * sizeof( uint32_t ) ); } char * Reader::getBlock( uint32_t address, vector< char > & chunk ) { size_t chunkIdx = address >> 16; if ( chunkIdx >= offsets.size() ) throw exAddressOutOfRange(); // Read and decompress the chunk { file.seek( offsets[ chunkIdx ] ); uint32_t uncompressedSize = file.read< uint32_t >(); uint32_t compressedSize = file.read< uint32_t >(); chunk.resize( uncompressedSize ); vector< unsigned char > compressedData( compressedSize ); file.read( &compressedData.front(), compressedData.size() ); unsigned long decompressedLength = chunk.size(); if ( uncompress( (unsigned char *)&chunk.front(), &decompressedLength, &compressedData.front(), compressedData.size() ) != Z_OK || decompressedLength != chunk.size() ) throw exFailedToDecompressChunk(); } size_t offsetInChunk = address & 0xffFF; if ( offsetInChunk > chunk.size() ) // It can be equal to for 0-sized blocks throw exAddressOutOfRange(); return &chunk.front() + offsetInChunk; } }