//---------------------------------------------------------------
// ____                   _ _         ____  _                    
//|  _ \ ___  _   _  __ _| | |_ _   _/ ___|| |__   __ _ _ __ ___ 
//| |_) / _ \| | | |/ _` | | __| | | \___ \| '_ \ / _` | '__/ _ \
//|  _ < (_) | |_| | (_| | | |_| |_| |___) | | | | (_| | | |  __/
//|_| \_\___/ \__, |\__,_|_|\__|\__, |____/|_| |_|\__,_|_|  \___|
//            |___/             |___/                            
//
// Copyright (C) 2011 RoyaltyShare, Inc.   All Rights Reserved
//---------------------------------------------------------------

#ifndef hPackedTable
#define hPackedTable

#include <sys/file.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <my_global.h>
#include <mysql.h>
#include <string.h>
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>


#define kErrorProgrammer 9999

#define kDataFileSuffix  "rs_ptf_WTF"

#if _FILE_OFFSET_BITS==64
typedef long long tOffset;          // This value will change with large files.  off_t seems broken, so be explicit
//#define kNullOffset ((tOffset)0xffffffffffffffff)
#define kNullOffset ((tOffset)0xffffffff)
#else
typedef long tOffset;          // This value will change with large files.  off_t seems broken, so be explicit
#define kNullOffset (tOffset)0xffffffff   
#endif
//
// This will need to be maintained if we change to 64 bit offsets.


// Declare some types 
typedef size_t tSize;          
typedef unsigned long tAddr; // Need to do some pointer math on void*s
typedef long tIndex;
typedef int tBool;


class PackedTable
{

protected:
    class DataFile;

    class DataBuffer
    {
    public:
        DataBuffer() {;}
        virtual ~DataBuffer() {;}

        void mapToFile(DataFile* file, int prot)
        {
            // How big is this file?
            //
            tOffset fileSize = file->size();

            // !!! Here is where I will care about file size, but
            // !!! for now assume it is < 2^32
            //
            // so, we'll need to have an array of void*s.
            // I'm not certain whether a memory address is signed or unsigned.  I'd think unsigned
            // 
            // This is _really_ feasable?   Can I _really_ mmap more than 1 big block?   How can the kernel
            // actually do that?  This is a 32-bit application...   What kind of process page scheme would 
            // support that?
            //
            // So, no, I don't think we can hope to map multiple MAX_32_INT() file regions at once.  So we won't
            // map them at once.   Instead, we will have to map one region at a time, and re-map as necessary.
            // -- This may cause some issues:  It's important to calculate our block sizes correctly to make sure
            //    each individual 'struct' lives within a single area.   The dynamic model complicates this somewhat.
            //
            // The standard (and minimum) page size is 4096.   So we can use that as a minimum quanta.
            // The MAX block size should be something considerably smaller than 2^32.    Might just go with 2^16 (65536)
            // Although that is rather dinky - We'll make it a constant, and test with 65536 so I can exercise this with smaller 
            // data sets.
            // 
            // ... Of course, I have an issue here.  I don't KNOW anything about struct sizes and stuff in this class.
            //     So, this hashing scheme will need to live upstream.
            //     Except I don't want to do that.   So I will need to provide this class with the block sizes.
            //     --> Might need to implement the index to offset to pointer interface completely in here.
            //         Easy, except for the dynamic classes.   In that case, I may be forced to do some weirdness.  
            //         For example, the block size may be required to be a union between the flat and dynamic data (so the
            //         largest becomes the block size).   Not very efficient, space-wise, but keeps the math easy.



            _dataSize = fileSize;
            tOffset offset = 0;
            _data = file->memmap(_dataSize, prot, offset);
        }

        void unmap(DataFile* file)
        {
            // How big is this file?
            //
            tSize fileSize = file->size();

            munmap(_data, fileSize);
        }


        // !!! Odd... so this also assumes a 32 bit address space.
        // !!! I will need to figure out where this gets called and convert it
        // !!! to pass tOffset types instead.
        // !!! Otherwise I will have a hard time iterating over big block boundaries.
        //
        // Returns true if the ptr falls within the memory space managed by this object.
        // 
        tBool ptrIsValid(void* ptr)
        {
            if (NULL == ptr)
                return 0;

            if ( ((tAddr)ptr >= (tAddr)_data) && ((tAddr)ptr < ((tAddr)_data + _dataSize)) )
                return 1;

            return 0;
        }

        // This is obviously the stupid 32-bit only version.
        // If I build this to support large files, off_t becomes a 64-bit value, and this math
        // will fail.
        void* ptr(tOffset offset) 
        { 
            return (void*) (((tAddr)_data) + ((tAddr)offset)); 
        }

    protected:
        void* _data;
        tOffset _dataSize;
    };



    class DataFile
    {
    protected:
        int _fd;
        char _fullPath[255];

        virtual const char* _outFileBasePath() = 0;
        virtual const char* _inFileBasePath()  = 0;
    public:
        DataFile() : _fd(0) { }
        virtual ~DataFile() { }

        tOffset size();  // Is this file size?  I guess it must be, so needs to be a tOffset
                        // !!! Search for usage.

        void openForWriting(const char* fileName);

        void openForReading(const char* fileName);

        void getWriteLock();
        
        void fill(tOffset newSize, char eraseByte);
                
        void closeFile();

        void appendData(void* dataPtr, size_t dataSize);

        virtual void* memmap(tSize size, int prot, tOffset offset)
        {
            void* data = this->_memmap(size, prot, offset);
            if (-1 == (tAddr)data || NULL == data)
            {
                fprintf(stderr, "ERROR calling mmap for %s: %s\n", _fullPath, strerror(errno));
                throw errno;
            }
            return data;
        }


    protected:
        virtual void* _memmap(tSize size, int prot, tOffset offset) = 0;
        
    };


    class VirtualMemoryFile : public DataFile
    {
    protected:
        virtual const char* _outFileBasePath() { return "/tmp"; }
        virtual const char* _inFileBasePath() { return "/tmp"; }
        virtual void* _memmap(tSize size, int prot, tOffset offset)
        {
            return mmap(NULL, size, prot, MAP_SHARED, _fd, offset);
        }

    public:
        virtual ~VirtualMemoryFile() { ; }
    };


    PackedTable::VirtualMemoryFile _dataFile;
    virtual DataFile* _theDataFile() { return &_dataFile; }


public:
    PackedTable() : _numItems(0) {;}
    virtual ~PackedTable() {;}

    virtual void createAndLoadDataFile(
     const char* fileBaseName, 
     const char* dbHost, 
     const char* dbUser, 
     const char* dbPassword, 
     const char* dbName );

    tIndex _numItems;
    virtual tIndex numItems() ;
    //
    // This returns the number of slots in the data array.
    // This does NOT tell you how many are 'real' - Just the size of the array.
    // NOT the number of bytes...
    // It also happens to be the max ID.
    // -- this could change, and probably will...


protected:
    virtual char _eraseByte() { return 0; }

    DataBuffer _dataBuffer;

    virtual void _createDataFile(const char* dbHost, const char* dbUser, const char* dbPassword, const char* dbName);

    virtual tIndex _queryNumElements(MYSQL* dbConnection);

    virtual void _queryReadData(MYSQL* dbConnection, DataBuffer* dataBuffer);

    static void _ParseLong(const char* inString, long* outLong);
    static void _ParseChar(const char* inString, char* outChar);
    static void _ParseByte(const char* inString, char* outByte);
    static void _ParseString(const char* inString, char* outString, tSize maxLen);
    static void _ParseDate(const char* inString, time_t* outTime);


    static const int kFieldPrimaryKey = 0; //  _By design_, the first item in always a primary key

    virtual void _writeRow(tIndex i, MYSQL_ROW row, DataBuffer* buffer);
    virtual void _writeRowFlat(tIndex i, MYSQL_ROW row, void* dataPtr) = 0;

    virtual const char* _dataQueryString() = 0;
    virtual const char* _maxIDQueryString() = 0;
    virtual tSize _dataFlatElementSize() = 0;

    virtual void* _getIndexedData(tIndex index);

    virtual void _loadDataFile(const char* fileBaseName);

    tIndex _idIterator;

public:
    virtual void resetIDIterator() {_idIterator = 0; }
    virtual tIndex nextID();
};



class PackedTableDynamic : public PackedTable
{
protected:
    struct _IDOffset
    {
        tIndex id;
        tOffset offset;
    };

    virtual char _eraseByte() { return 0xff; }
    virtual tSize _dataFlatElementSize() { return sizeof(struct _IDOffset); }


    // The stl classes are proving to be a pain in the ass.
    // map seems ok, but the list class is giving me fits.
    // I'm just going to implement my own generic linked list class, which means
    // that I can implement more of this in the base class (just going to use memcpy and stuff).
    //
    class _List
    {
    protected:
        struct _node {
            void* dataPtr;
            _node* nextNode;
            tSize dataSize;

            _node() : dataPtr(NULL), nextNode(NULL), dataSize(0) {;}
            ~_node() { if (NULL != dataPtr) free(dataPtr); }
        };

        struct _node* _head;
    public:
        _List() : _head(NULL) {;}
        ~_List()
        {
            while (NULL != _head)
            {
                this->pop(NULL);
            }
        }

        void push(void* inData, tSize dataSize)
        {
            // Add to the front.
            //
            struct _node* newNode = new struct _node;
            newNode->dataPtr = malloc(dataSize);
            newNode->dataSize = dataSize;
            memcpy(newNode->dataPtr, inData, dataSize);
            newNode->nextNode = _head;
            _head = newNode;
        }

        tBool pop(void* outData)
        {
            if (NULL == _head)
            {
                return 0;
            }

            // Remove from front. 
            // Copy the data if we were passed an address.
            //
            if (NULL != outData)
            {
                memcpy(outData, _head->dataPtr, _head->dataSize);
            }
            struct _node* oldHead = _head;
            _head = oldHead->nextNode;
            delete oldHead;

            return 1;
        }
    };

protected:
    _List _theList;
    _List* _theListPtr() { return &_theList; }

    tOffset _lastDynamicOffset;
    tIndex _lastDynamicID;
    void* _dynamicItemBuffer;

    virtual void _writeRow(tIndex i, MYSQL_ROW row, DataBuffer* dataBuffer);
    virtual tIndex _writeDynamicList(DataBuffer* dataBuffer);

    virtual tSize _dataDynamicElementSize() = 0;
    virtual tIndex _writeRowDynamic(tIndex i, MYSQL_ROW row, void* dataPtr) = 0;
    virtual void _writeRowFlat(tIndex i, MYSQL_ROW row, void* dataPtr);

    virtual void _pushDynamicList(void* dataPtr)
    {
        this->_theListPtr()->push(dataPtr, this->_dataDynamicElementSize());
    }
    virtual tBool _popDynamicList(void* dataPtr)
    {
        return this->_theListPtr()->pop(dataPtr);
    }

    virtual void _queryReadData(MYSQL* dbConnection, DataBuffer* dataBuffer);

public:
    PackedTableDynamic() : _lastDynamicID(0), _lastDynamicOffset(0), _dynamicItemBuffer(NULL), _theIterator(this), PackedTable() {;}
    virtual ~PackedTableDynamic() 
    {
        if (NULL != _dynamicItemBuffer)
        {
            free(_dynamicItemBuffer);
            _dynamicItemBuffer = NULL;
        }
    }

    virtual tIndex numItems(); 

    void setIteratorAtIndex(tIndex index)
    {
        if (index < 0 || index > (this->numItems() - 1))
        {
            _theIterator._ptr = NULL;
            return;
        }

        struct _IDOffset* indexBlock = (struct _IDOffset*)this->_getIndexedData(index);
        if (NULL == indexBlock)
        {
            _theIterator._ptr = NULL;
            return;
        }
        else
        {
            if ((tOffset)kNullOffset == (tOffset)indexBlock->offset)
            {
                _theIterator._ptr = NULL;
                return;
            }

            // !!! Actually set the pointer to the offset.
            // !!! In the New Scheme, everything is in the same place.
            // !!! So ptr should equal the starting ptr plus the ofset.
            _theIterator._ptr = _dataBuffer.ptr(indexBlock->offset);
        }
    }

protected:
    tBool _pointerIsValid(void* ptr)
    {
        if (NULL == ptr)
            return 0;

        // The dataBuffer object will manage all the addresses.
        // So let it figure out if ptr is in range.
        //
        return _dataBuffer.ptrIsValid(ptr);
    }

    virtual tBool _equalKeys(void* ptrA, void* ptrB) = 0;


    class Iterator
    {
    public:
        PackedTableDynamic* _table;
        void* _ptr;

        Iterator(PackedTableDynamic* table) : _table(table), _ptr(NULL) {;}
        virtual ~Iterator() {;}

        tBool hasNext()
        {
            if (this->_ptr != NULL)
                return 1;
            return 0;
        }

        void* next()
        {
            // We'll return whatever the stored pointer already is.
            //
            void* returnPtr = this->_ptr;

            // Increment the stored pointer if possible.
            //
            if (this->_ptr != NULL)
            {
                tAddr addr = (tAddr)this->_ptr;
                addr += _table->_dataDynamicElementSize();
                this->_ptr = (void*)addr;

                if (! _table->_pointerIsValid(this->_ptr))
                {
                    // Ran off the table.
                    //
                    this->_ptr = NULL;
                }
                else
                {
                    // Reached the end of this list of items.
                    //
                    if (! _table->_equalKeys(returnPtr, this->_ptr))
                    {
                        this->_ptr = NULL;
                    }
                }
            }

            return returnPtr;
        }
    };

    Iterator _theIterator;
};


#endif