//--------------------------------------------------------------- // ____ _ _ ____ _ //| _ \ ___ _ _ __ _| | |_ _ _/ ___|| |__ __ _ _ __ ___ //| |_) / _ \| | | |/ _` | | __| | | \___ \| '_ \ / _` | '__/ _ \ //| _ < (_) | |_| | (_| | | |_| |_| |___) | | | | (_| | | | __/ //|_| \_\___/ \__, |\__,_|_|\__|\__, |____/|_| |_|\__,_|_| \___| // |___/ |___/ // // Copyright (C) 2011 RoyaltyShare, Inc. All Rights Reserved //--------------------------------------------------------------- #include #include #include #include #include #include #include #include #include #include #include "PackedTable.h" void PackedTable::_ParseLong(const char* inString, long* outLong) { if (NULL == inString) { *outLong = 0; } else { sscanf(inString, "%ld", outLong); } } void PackedTable::_ParseChar(const char* inString, char* outChar) { if (NULL == inString) { *outChar= 0; } else { sscanf(inString, "%c", outChar); } } void PackedTable::_ParseByte(const char* inString, char* outByte) { if (NULL == inString) { *outByte = 0; } else { sscanf(inString, "%d", outByte); } } void PackedTable::_ParseString(const char* inString, char* outString, tSize maxLen) { if (NULL == inString) { strcpy(outString, ""); } else { // Make sure the string is NULL terminated by setting it to 0 first. memset(outString, 0, maxLen); strncpy(outString, inString, maxLen - 1); } } void PackedTable::_ParseDate(const char* inString, time_t* outTime) { if (NULL == inString) { *outTime = 0; } else { sscanf(inString, "%d", outTime); } } tOffset PackedTable::DataFile::size() { struct stat fStat; if (-1 == fstat(_fd, &fStat)) { fprintf(stderr, "ERROR reading file size of %s: %s\n", _fullPath, strerror(errno)); throw errno; } return fStat.st_size; } void PackedTable::DataFile::openForWriting(const char* fileName) { const char* basePath = this->_outFileBasePath(); sprintf(_fullPath, "%s/%s.%s", basePath, fileName, kDataFileSuffix); _fd = open(_fullPath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); if (-1 == _fd) { fprintf(stderr, "ERROR - unable to open %s for writing: errno:%d\n", _fullPath, errno); throw errno; } } void PackedTable::DataFile::openForReading(const char* fileName) { sprintf(_fullPath, "%s/%s.%s", this->_inFileBasePath(), fileName, kDataFileSuffix); _fd = open(_fullPath, O_RDONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); if (-1 == _fd) { fprintf(stderr, "PackedTable::DataFile::openForReading ERROR - unable to open %s for reading: errno:%d\n", _fullPath, errno); throw errno; } } void PackedTable::DataFile::getWriteLock() { if (-1 == flock(_fd, LOCK_EX)) { perror("ERROR obtaining write lock"); throw errno; } } void PackedTable::DataFile::fill(tOffset totalBlockSize, char eraseByte) { const tSize kBlockSize = 2048; char cBuf[kBlockSize]; memset(cBuf, eraseByte, kBlockSize); tOffset remainingBytes = totalBlockSize; while (remainingBytes > 0) { // !!! Don't think I need to typecase different integer types to do math, do I? // !!! As long as I am not assigning a 64 to a 32 bit entity. // tSize bytesToWrite = kBlockSize; if (remainingBytes < kBlockSize) { bytesToWrite = remainingBytes; } if (-1 == write(_fd, cBuf, bytesToWrite)) { fprintf(stderr, "ERROR - error writing to %s: errno:%d\n", _fullPath, errno); throw errno; } remainingBytes -= bytesToWrite; } } void PackedTable::DataFile::closeFile() { close(_fd); _fd = -1; } void PackedTable::DataFile::appendData(void* dataPtr, tSize dataSize) { // The seek to the end of the might be superfluous, but seems harmless enough. // if (-1 == lseek(_fd, 0, SEEK_END)) { fprintf(stderr, "appendData ERROR - error seeking to end of %s: errno:%d\n", _fullPath, errno); fprintf(stderr, "_fd : %d\n", _fd); throw errno; } if (-1 == write(_fd, dataPtr, dataSize)) { fprintf(stderr, "appendData ERROR - error writing to %s: errno:%d\n", _fullPath, errno); throw errno; } } void PackedTable::createAndLoadDataFile( const char* fileBaseName, const char* dbHost, const char* dbUser, const char* dbPassword, const char* dbName ) { this->_theDataFile()->openForWriting(fileBaseName); this->_theDataFile()->getWriteLock(); // If the file is empty, then we must create it. // tSize dataSize = this->_theDataFile()->size(); if (0 == dataSize) { this->_createDataFile(dbHost, dbUser, dbPassword, dbName); } // The lock is released when the file is closed. // this->_theDataFile()->closeFile(); // Now we re-open the file for reading, and load it. // this->_loadDataFile(fileBaseName); } long PackedTable::numItems() { if (0 == _numItems) { _numItems = this->_theDataFile()->size() / this->_dataFlatElementSize(); } return _numItems; } void PackedTable::_createDataFile(const char* dbHost, const char* dbUser, const char* dbPassword, const char* dbName) { MYSQL* dbConnection = mysql_init(NULL); mysql_real_connect(dbConnection, dbHost, dbUser, dbPassword, dbName, 0, NULL, 0); tIndex numElements = this->_queryNumElements(dbConnection); tSize dataElementSize = this->_dataFlatElementSize(); // Allocating an extra block because ids are 1-based. // !!! This is te first place we can go awry with big files. // !!! Do I need to cast for this math to work out? // !!! Let's try it. // tOffset totalBlockSize = (tOffset)(numElements + 1) * (tOffset)dataElementSize; // The data file should already be open for writing. // this->_theDataFile()->fill(totalBlockSize, this->_eraseByte()); DataBuffer dataBuffer; dataBuffer.mapToFile(this->_theDataFile(), PROT_READ | PROT_WRITE); this->_queryReadData(dbConnection, &dataBuffer); // We're done, in theory, so release the mmap. // dataBuffer.unmap(this->_theDataFile()); } tIndex PackedTable::_queryNumElements(MYSQL* dbConnection) { // The number of 'elements' (array slots, if you will), will be equal to the // MAX(id) of the table we're flattening. // int error = mysql_query(dbConnection, this->_maxIDQueryString()); if (error) { fprintf(stderr, "ERROR - mysql_query error:%d - %s\n", error, mysql_error(dbConnection)); throw error; } MYSQL_RES* result = mysql_store_result(dbConnection); if (NULL == result) { error = mysql_errno(dbConnection); fprintf(stderr, "ERROR - mysql_store_result error:%d\n", error); throw error; } MYSQL_ROW row = mysql_fetch_row(result); error = mysql_errno(dbConnection); if (error) { fprintf(stderr, "ERROR - mysql_fetch_row error:%d\n", error); throw error; } tIndex numElements; PackedTable::_ParseLong(row[0], &numElements); mysql_free_result(result); return numElements; } void PackedTable::_queryReadData(MYSQL* dbConnection, DataBuffer* dataBuffer) { int error = mysql_query(dbConnection, this->_dataQueryString()); if (error) { fprintf(stderr, "ERROR - mysql_query error:%d - %s\n", error, mysql_error(dbConnection)); throw error; } MYSQL_RES* result = mysql_use_result(dbConnection); if (NULL == result) { error = mysql_errno(dbConnection); fprintf(stderr, "ERROR - mysql_store_result error:%d\n", error); throw error; } MYSQL_ROW row; while (NULL != (row = mysql_fetch_row(result))) { // _writeRow is implemented in each subclass. // // Read the primary key out of the mysql row // tIndex i; PackedTable::_ParseLong(row[kFieldPrimaryKey], &i); this->_writeRow(i, row, dataBuffer); } error = mysql_errno(dbConnection); if (error) { fprintf(stderr, "ERROR - mysql_fetch_row error:%d\n", error); throw error; } mysql_free_result(result); } void PackedTable::_writeRow(tIndex i, MYSQL_ROW row, DataBuffer* dataBuffer) { // Translate that key into an offset (i.e. relative address within the data buffer) // Then fetch a pointer to that address. // tOffset offset = (tOffset)i * (tOffset)this->_dataFlatElementSize(); void* dataPtr = dataBuffer->ptr(offset); this->_writeRowFlat(i, row, dataPtr); } void* PackedTable::_getIndexedData(const tIndex index) { if (index > (this->numItems() - 1)) { return NULL; } // Casting everything to tOffset, so it all will work if we end up with // a 64-bit offset. // void* dataPtr = _dataBuffer.ptr((tOffset)index * (tOffset)this->_dataFlatElementSize()); return dataPtr; } void PackedTable::_loadDataFile(const char* fileBaseName) { // All we should need to do to 'load' the file is map it. // this->_theDataFile()->openForReading(fileBaseName); _dataBuffer.mapToFile(this->_theDataFile(), PROT_READ); } long PackedTable::nextID() { tIndex maxID = this->numItems() - 1; while (++_idIterator <= maxID) { void* dataPtr = this->_getIndexedData(_idIterator); if (NULL == dataPtr) return 0; // !!! This is a serious hack. We assume that the first item is a long, which is the index. // !!! That's been the case so far, and makes this check cheap and easy. If we change that, though, // !!! then all hell will break loose... // // If this is a valid data block, then this is the next id // And we'll know it is valid if the id (the first long) is the same // tIndex* idPtr = (tIndex*)dataPtr; if (*idPtr == _idIterator) { return _idIterator; } } return 0; } void PackedTableDynamic::_writeRow(tIndex i, MYSQL_ROW row, DataBuffer* dataBuffer) { // The first time through, I'll need to figure some stuff out. // - Need to know just how big the index data block is, so I can set // up the offsets properly. I guess at this point that is the same // as the size of the file, so I can use that. // if (0 == _lastDynamicOffset) { _lastDynamicID = 0; _lastDynamicOffset = this->_theDataFile()->size(); tSize size = this->_dataDynamicElementSize(); void* buffer = (void*)malloc( size ); this->_dynamicItemBuffer = buffer; // !!! Can't call getIndexedData here... That will fail horibly. // So we'll do this by hand, so to speak. // // struct _IDOffset* firstRecord = (struct _IDOffset*)this->_getIndexedData(0); struct _IDOffset* firstRecord = (struct _IDOffset*)dataBuffer->ptr(0); // Record the size of the index block in the first record. // (*firstRecord).offset = _lastDynamicOffset; } // First we're going to call a new method, _writeDynamicData(), which will parse // the row and returns the id. this->_writeRowDynamic(i, row, (void*)_dynamicItemBuffer); if (0 != _lastDynamicID && i != _lastDynamicID) { this->_writeDynamicList(dataBuffer); } _lastDynamicID = i; this->_pushDynamicList((void*)_dynamicItemBuffer); } void PackedTableDynamic::_queryReadData(MYSQL* dbConnection, DataBuffer* dataBuffer) { // Call the inherited method first. // PackedTable::_queryReadData(dbConnection, dataBuffer); // Output the final dynamic list, if we've built any. // if (0 != _lastDynamicID) { this->_writeDynamicList(dataBuffer); } } void PackedTableDynamic::_writeRowFlat(tIndex i, MYSQL_ROW row, void* data) { struct _IDOffset* dataPtr = (struct _IDOffset*)data; (*dataPtr).id = _lastDynamicID; (*dataPtr).offset = _lastDynamicOffset; } tIndex PackedTableDynamic::_writeDynamicList(DataBuffer* dataBuffer) { // We want to seek to the end of the file, and write out all the data // !!! If I had a pointer to the container, I think I would be good to go. // !!! Maybe. tOffset dataWritten = 0; while (this->_popDynamicList(_dynamicItemBuffer)) { this->_theDataFile()->appendData(_dynamicItemBuffer, this->_dataDynamicElementSize()); dataWritten += this->_dataDynamicElementSize(); } if (0 != dataWritten) { // Write to the flat index to offset map. // !!! Need to be carefull, since the _lastDynamicID is NOT THE SAME ID as // !!! the one in the current row. // tOffset offset = (tOffset)_lastDynamicID * (tOffset)this->_dataFlatElementSize(); void* dataPtr = dataBuffer->ptr(offset); this->_writeRowFlat(_lastDynamicID, (MYSQL_ROW)NULL, dataPtr); _lastDynamicOffset += dataWritten; } } tIndex PackedTableDynamic::numItems() { if (0 == _numItems) { // Remember, cannot use getIndexedData because that in turn calls this method. // struct _IDOffset* firstRecord = (struct _IDOffset*)_dataBuffer.ptr(0); // !!! Casting from a potential 64 bit value to 32 // _numItems = (tIndex)(*firstRecord).offset; } return _numItems; }