synchronized_texture_heap_opencl.h

#pragma once

#ifndef SYNCHRONIZED_TEXTURE_HEAP_OPENCL_H
#define SYNCHRONIZED_TEXTURE_HEAP_OPENCL_H




#include "settings.h"




#if ( SETTING_OPENCL != 0 )




#include "synchronized_texture_heap_base.h"
#include "synchronized_detail_octree.h"
#include "platform.h"
#include "color.h"
#include "vector.h"
#include "opencl.h"

#include <unordered_set>
#include <memory>
#include <array>

#include <CL/cl.h>




namespace Synchronized {




/*==============================================================================
    TextureHeap< PLATFORM_OPENCL > specialization
==============================================================================*/


template< int t_ColorFormat >
struct TextureHeap< t_ColorFormat, PLATFORM_OPENCL > {

    struct Pointer;

    typedef cl_mem OpenCLType;


    inline TextureHeap( std::shared_ptr< OpenCL::State > const& pState, unsigned int const logSize );


    inline Pointer Allocate( unsigned int const logSize );


    inline void Synchronize();


    inline OpenCLType OpenCLConvert() const;


private:

    struct Object {

        inline Object( TextureHeap* const pParent, Vector< unsigned int, 3 > const& coordinates, unsigned int const logSize );

        inline ~Object();


        inline void OpenCLWrite( void* const destination, size_t const rowPitch, size_t const slicePitch ) const;


        inline unsigned int References() const;

        inline void AddReference();
        inline void ReleaseReference();


        inline Vector< unsigned int, 3 > const& Coordinates() const;
        inline unsigned int const LogSize() const;

        inline void Mark() const;


        inline Color< t_ColorFormat > const* Get( size_t const index = 0 ) const;
        inline void Set( size_t const index, Color< t_ColorFormat > const& value );


    private:

        unsigned int m_refcount;

        TextureHeap* m_pParent;
        Vector< unsigned int, 3 > m_coordinates;
        unsigned int m_logSize;

        Color< t_ColorFormat >* m_array;
    };


public:

    struct Pointer {

        typedef Color< t_ColorFormat > const element_type;
        typedef cl_uint OpenCLType;    // **NOTE: (x,y,z) coordinates are packed into (11,11,10)-bit segments


        inline Pointer();
        inline Pointer( Pointer const& other );

        inline ~Pointer();


        inline void reset();
        inline void swap( Pointer& other );

        inline unsigned int use_count() const;
        inline bool unique() const;


        inline Color< t_ColorFormat > const* get() const;

        inline void Assign( size_t const index, Color< t_ColorFormat > const& data ) const;


        inline Pointer const& operator=( Pointer const& other );


        inline operator bool() const;

        inline Color< t_ColorFormat > const* operator->() const;
        inline Color< t_ColorFormat > const& operator*() const;

        inline Color< t_ColorFormat > const& operator[]( size_t const index ) const;

        inline bool operator==( Pointer const& other ) const;
        inline bool operator!=( Pointer const& other ) const;


        inline OpenCLType OpenCLConvert() const;


    private:

        inline Pointer( Object* pointer );


        Object* m_pointer;


        friend struct TextureHeap< t_ColorFormat, PLATFORM_OPENCL >;
    };


private:

    // sorts in Morton Z-order so that marked objects are listed in the same order as detail::Octree leaves
    struct ObjectComparison {

        inline bool operator()( Object const* const pLeft, Object const* const pRight ) const;
    };


    inline void FreeObject( Object const* pObject );

    inline void MarkObject( Object const* pObject );


    typedef std::unordered_set< Object const* > MarkedObjects;


    std::shared_ptr< OpenCL::State > m_pState;
    std::unique_ptr< _cl_mem, OpenCL::Deleter< _cl_mem > > m_image;

    detail::Octree m_freeTree;
    MarkedObjects m_markedObjects;
};




/*==============================================================================
    TextureHeap< PLATFORM_OPENCL >::Object methods
==============================================================================*/


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::Object( TextureHeap* const pParent, Vector< unsigned int, 3 > const& coordinates, unsigned int const logSize ) :
    m_refcount( 0 ),
    m_pParent( pParent ),
    m_coordinates( coordinates ),
    m_logSize( logSize ),
    m_array( new Color< t_ColorFormat >[ 1u << ( 3 * m_logSize ) ] )
{
}


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::~Object() {

    assert( m_refcount == 0 );
    assert( m_pParent != NULL );
    m_pParent->FreeObject( this );
    delete[] m_array;
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::OpenCLWrite( void* const destination, size_t const rowPitch, size_t const slicePitch ) const {

    typedef typename OpenCL::Converter< Color< t_ColorFormat > >::Type OpenCLType;

    Color< t_ColorFormat > const* kk = m_array;
    uint8_t* iiDestination = reinterpret_cast< uint8_t* >( destination );
    for ( unsigned int ii = 0; ii < ( 1u << m_logSize ); ++ii ) {

        uint8_t* jjDestination = iiDestination;
        for ( unsigned int jj = 0; jj < ( 1u << m_logSize ); ++jj ) {

            OpenCLType* kkDestination    = reinterpret_cast< OpenCLType* >( jjDestination );
            OpenCLType* kkDestinationEnd = kkDestination + ( 1u << m_logSize );
            for ( ; kkDestination != kkDestinationEnd; ++kkDestination, ++kk )
                *kkDestination = OpenCL::Converter< Color< t_ColorFormat > >::Convert( *kk );
            jjDestination += rowPitch;
        }

        iiDestination += slicePitch;
    }
}


template< int t_ColorFormat >
unsigned int TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::References() const {

    return m_refcount;
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::AddReference() {

    ++m_refcount;
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::ReleaseReference() {

    assert( m_refcount > 0 );
    if ( --m_refcount == 0 )
        delete this;
}


template< int t_ColorFormat >
Vector< unsigned int, 3 > const& TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::Coordinates() const {

    return m_coordinates;
}


template< int t_ColorFormat >
unsigned int const TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::LogSize() const {

    return m_logSize;
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::Mark() const {

    assert( m_pParent != NULL );
    m_pParent->MarkObject( this );
}


template< int t_ColorFormat >
Color< t_ColorFormat > const* TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::Get( size_t const index ) const {

    assert( index < ( 1u << ( 3 * m_logSize ) ) );
    return( m_array + index );
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Object::Set( size_t const index, Color< t_ColorFormat > const& data ) {

    assert( index < ( 1u << ( 3 * m_logSize ) ) );
    m_array[ index ] = data;
    Mark();
}




/*==============================================================================
    TextureHeap< PLATFORM_OPENCL >::Pointer methods
==============================================================================*/


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::Pointer() : m_pointer( NULL ) {
}


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::Pointer( Pointer const& other ) : m_pointer( other.m_pointer ) {

    if ( m_pointer != NULL )
        m_pointer->AddReference();
}


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::~Pointer() {

    reset();
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::reset() {

    if ( m_pointer != NULL ) {

        m_pointer->ReleaseReference();
        m_pointer = NULL;
    }
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::swap( Pointer& other ) {

    std::swap( m_pointer, other.m_pointer );
}


template< int t_ColorFormat >
unsigned int TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::use_count() const {

    unsigned int result = 0;
    if ( m_pointer != NULL )
        result = m_pointer->References();
    return result;
}


template< int t_ColorFormat >
bool TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::unique() const {

    return( use_count() == 1 );
}


template< int t_ColorFormat >
Color< t_ColorFormat > const* TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::get() const {

    return m_pointer->Get();
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::Assign( size_t const index, Color< t_ColorFormat > const& data ) const {

    m_pointer->Set( index, data );
}


template< int t_ColorFormat >
typename TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer const& TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator=( Pointer const& other ) {

    if ( m_pointer != other.m_pointer ) {

        reset();
        m_pointer = other.m_pointer;
        if ( m_pointer != NULL )
            m_pointer->AddReference();
    }

    return *this;
}


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator bool() const {

    return( m_pointer != NULL );
}


template< int t_ColorFormat >
Color< t_ColorFormat > const* TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator->() const {

    return m_pointer->Get();
}


template< int t_ColorFormat >
Color< t_ColorFormat > const& TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator*() const {

    return *m_pointer->Get();
}


template< int t_ColorFormat >
Color< t_ColorFormat > const& TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator[]( size_t const index ) const {

    return *m_pointer->Get( index );
}


template< int t_ColorFormat >
bool TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator==( Pointer const& other ) const {

    return( m_pointer == other.m_pointer );
}


template< int t_ColorFormat >
bool TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::operator!=( Pointer const& other ) const {

    return( m_pointer != other.m_pointer );
}


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::Pointer( Object* pointer ) : m_pointer( pointer ) {

    if ( m_pointer != NULL )
        m_pointer->AddReference();
}


template< int t_ColorFormat >
typename TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::OpenCLType TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::OpenCLConvert() const {

    TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer::OpenCLType result = OPENCL_NULL;
    if ( m_pointer != NULL ) {

        Vector< unsigned int, 3 > coordinates = m_pointer->Coordinates();
        assert( coordinates[ 0 ] < ( 1u << 11 ) );    // **NOTE: x-coordinate = 11 bits
        assert( coordinates[ 1 ] < ( 1u << 11 ) );    // **NOTE: y-coordinate = 11 bits
        assert( coordinates[ 2 ] < ( 1u << 10 ) );    // **NOTE: z-coordinate = 10 bits
        result = ( ( ( coordinates[ 2 ] << 11 ) + coordinates[ 1 ] ) << 11 ) + coordinates[ 0 ];
    }
    return result;
}




/*==============================================================================
    TextureHeap< PLATFORM_OPENCL >::ObjectComparison methods
==============================================================================*/


template< int t_ColorFormat >
bool TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::ObjectComparison::operator()( Object const* const pLeft, Object const* const pRight ) const {

    int comparison = 0;

    Vector< unsigned int, 3 > const& leftCoordinates = pLeft->Coordinates();
    assert( leftCoordinates[ 0 ] < ( 1u << 11 ) );    // **NOTE: x-coordinate = 11 bits
    assert( leftCoordinates[ 1 ] < ( 1u << 11 ) );    // **NOTE: y-coordinate = 11 bits
    assert( leftCoordinates[ 2 ] < ( 1u << 10 ) );    // **NOTE: z-coordinate = 10 bits

    Vector< unsigned int, 3 > const& rightCoordinates = pRight->Coordinates();
    assert( rightCoordinates[ 0 ] < ( 1u << 11 ) );    // **NOTE: x-coordinate = 11 bits
    assert( rightCoordinates[ 1 ] < ( 1u << 11 ) );    // **NOTE: y-coordinate = 11 bits
    assert( rightCoordinates[ 2 ] < ( 1u << 10 ) );    // **NOTE: z-coordinate = 10 bits

    for ( int ii = 10; ( comparison == 0 ) && ( ii >= 0 ); --ii ) {    // **NOTE: all coordinates must be in the range [0,2^11)

        unsigned int const mask = ( 1u << ii );

        for ( int jj = 2; jj >= 0; --jj ) {

            unsigned int const left  = ( leftCoordinates[  jj ] & mask );
            unsigned int const right = ( rightCoordinates[ jj ] & mask );
            if ( left < right ) {

                comparison = -1;
                break;    // **NOTE: break
            }
            else if ( left > right ) {

                comparison = 1;
                break;    // **NOTE: break
            }
        }
    }

    return( comparison < 0 );
}




/*==============================================================================
    TextureHeap< PLATFORM_OPENCL > methods
==============================================================================*/


template< int t_ColorFormat >
TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::TextureHeap( std::shared_ptr< OpenCL::State > const& pState, unsigned int const logSize ) : m_pState( pState ) {

    assert( logSize <= 10 );    // **NOTE: (x,y,z) coordinates are packed into (11,11,10)-bit segments

    Vector< size_t, 3 > const size = m_freeTree.Fill( logSize );

    cl_int result = CL_SUCCESS;
    m_image.reset( clCreateImage3D( m_pState->Context(), CL_MEM_READ_ONLY | CL_MEM_ALLOC_HOST_PTR, &OpenCL::ImageFormat< t_ColorFormat >::format, size[ 0 ], size[ 1 ], size[ 2 ], 0, 0, NULL, &result ) );
    if ( result != CL_SUCCESS )
        THROW_OPENCL_EXCEPTION( "clCreateImage3D failed!", result );
}


template< int t_ColorFormat >
typename TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Pointer TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Allocate( unsigned int const logSize ) {

    Vector< unsigned int, 3 > coordinates = m_freeTree.Erase( logSize );
    Object* pObject = new Object( this, coordinates, logSize );
    MarkObject( pObject );
    return Pointer( pObject );
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::Synchronize() {

    if ( ! m_markedObjects.empty() ) {

        typename MarkedObjects::size_type const size = m_markedObjects.size();
        std::unique_ptr< Object const* > markedObjects( new Object const*[ size ] );
        std::copy( m_markedObjects.begin(), m_markedObjects.end(), markedObjects.get() );
        std::sort( markedObjects.get(), markedObjects.get() + size, ObjectComparison() );
        Object const* const* const iiBegin = markedObjects.get();
        Object const* const* const iiEnd   = markedObjects.get() + size;

        detail::Octree markedTree( m_freeTree.Height() );
        for ( Object const* const* ii = iiBegin; ii != iiEnd; ++ii )
            markedTree.Insert( ( *ii )->Coordinates(), ( *ii )->LogSize() );

        typedef std::pair< Vector< unsigned int, 3 >, unsigned int > Leaf;
        unsigned int const leaves = markedTree.CountLeaves();
        std::unique_ptr< Leaf > markedLeaves( new Leaf[ leaves ] );
        markedTree.Leaves( markedLeaves.get() );
        Leaf const* const jjBegin = markedLeaves.get();
        Leaf const* const jjEnd   = markedLeaves.get() + leaves;

        Object const* const* ii = iiBegin;
        Leaf const* jj = jjBegin;
        for ( ; jj != jjEnd; ++jj ) {

            assert( ii != iiEnd );

            unsigned int const mask = ( 1u << jj->second ) - 1;

            size_t origin[ 3 ] = { jj->first[ 0 ], jj->first[ 1 ], jj->first[ 2 ] };
            size_t region[ 3 ] = { ( 1u << jj->second ), ( 1u << jj->second ), ( 1u << jj->second ) };
            size_t rowPitch   = 0;
            size_t slicePitch = 0;

            cl_int result = CL_SUCCESS;
            void* const buffer = clEnqueueMapImage(
                m_pState->Queue(),
                m_image.get(),
                CL_TRUE,    // blocking
                CL_MAP_WRITE,
                origin,
                region,
                &rowPitch,
                &slicePitch,
                0,
                NULL,
                NULL,
                &result
            );
            if ( result != CL_SUCCESS )
                THROW_OPENCL_EXCEPTION( "clEnqueueMapImage failed!", result );

            Object const* const* iiNext = ii;
            for ( ; iiNext != iiEnd; ++iiNext ) {

                Vector< unsigned int, 3 > const& coordinates = ( *iiNext )->Coordinates();
                if (
                    ( ( coordinates[ 0 ] & ~mask ) != jj->first[ 0 ] ) ||
                    ( ( coordinates[ 1 ] & ~mask ) != jj->first[ 1 ] ) ||
                    ( ( coordinates[ 2 ] & ~mask ) != jj->first[ 2 ] )
                )
                {
                    break;
                }
            }
            assert( ii != iiNext );

            for ( ; ii != iiNext; ++ii ) {

                Vector< unsigned int, 3 > const& coordinates = ( *ii )->Coordinates();
                uint8_t* const destination = (
                    reinterpret_cast< uint8_t* >( buffer ) +
                    ( coordinates[ 2 ] & mask ) * slicePitch +
                    ( coordinates[ 1 ] & mask ) * rowPitch +
                    ( coordinates[ 0 ] & mask ) * sizeof( typename OpenCL::Converter< Color< t_ColorFormat > >::Type )
                );
                ( *ii )->OpenCLWrite( destination, rowPitch, slicePitch );
            }

            clEnqueueUnmapMemObject(
                m_pState->Queue(),
                m_image.get(),
                buffer,
                0,
                NULL,
                NULL
            );
        }
        assert( ii == iiEnd );

        m_markedObjects.clear();
    }
}


template< int t_ColorFormat >
typename TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::OpenCLType TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::OpenCLConvert() const {

    return m_image.get();
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::FreeObject( Object const* pObject ) {

    m_markedObjects.erase( pObject );
    m_freeTree.Insert( pObject->Coordinates(), pObject->LogSize() );
}


template< int t_ColorFormat >
void TextureHeap< t_ColorFormat, PLATFORM_OPENCL >::MarkObject( Object const* pObject ) {

    m_markedObjects.insert( pObject );
}




}    // namespace Synchronized




#endif    // SETTING_OPENCL




#endif    // SYNCHRONIZED_TEXTURE_HEAP_OPENCL_H