/*
 * threads.c
 *
 * x-kernel v3.3
 *
 * Copyright (c) 1993,1991,1990,1996  Arizona Board of Regents
 *
 * $Log: thread.c,v $
 * Revision 1.1  1996/11/22 15:48:40  dorgival
 * Initial revision
 *
 *
 */

/*
 * Thread handling for the x-kernel running in user-mode in the Paragon.
 *
 * In this case we are using the OSF-1-provided P-threads package
 * and implementing the basic functions needed by other modules and
 * protocols. There is not provision right now for the special needs
 * of device drivers and other threads external to the x-kernel.
 * They should either use P-threads directly or build their interface.
 */

/*
 * The first version of this module controlled access to the x-kernel
 * with only one mutex variable. Processes trying to enter the kernel
 * just tried to get a lock on the mutex. It turns out the way locking
 * is implemented (by busy waiting+yielding) this is not a good idea.
 * The right way to do it is by making processes to wait on (at least
 * one) condition variable, when they are suspended for good.
 * On controlling the access it may be a good idea to use different
 * condition variables for different types of threads: external,
 * threads suspended in the x-kernel, and shepherd threads are
 * three cathegories so far. Then we can assign different priorities
 * for lock the kernel.
 */

#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <sys/timers.h>
#include <string.h>

#include "/usr/include/pthread.h"

#include "event.h"
#include "event_i.h"
#include "xk_thread.h"
#include "trace.h"
#include "xk_debug.h"
#include "x_util.h"

#define SUGG_MAX_THREAD    6 /* The manuals suggest never to have more than 6 */
#define DEFAULT_STACK_SIZE      (16*INIT_STACK_SIZE)

int tracethreads;

int    xk_thread_count, p_thread_count;
static pthread_attr_t  xk_thread_attr;

/*
 * xk_lock is used to guarantee access to the x-kernel is done on
 * a mutual exclusion basis. There are three levels of access: Every
 * time control of the x-kernel is relinguished and may be assigned
 * to another thread, first an internal queue of locked threads is
 * verified and if a thread is found in it, it gets the kernel. Otherwise
 * any thread in a lock for shepherd threads is waked, and finally
 * external threads which require access to the kernel.
 *
 * As it is, this scheme should give priority to threads which were
 * blocked inside the kernel - waiting on an event, suspended on a
 * semaphore, etc. After that, it privileges shepherd threads, which
 * theoretically will attempt to enter the kernel to process incoming
 * packets, with high priority, and finally it deals with external
 * threads with attempt to lock the x-kernel. This may include
 * threads from anchor protocols, usually pushing messages down the stack.
 *
 * It is still to be verified if this scheme is fair enough, or if the
 * priotities policy may create starvation scenarios which may reduce
 * overall performance.
 */

struct {
    xk_thread_t*    owner;
    int             access;
    int             internal_locked;
    int             shepherd_locked;
    pthread_mutex_t mutex;
    pthread_cond_t  external_cond;
    pthread_cond_t  internal_cond;
    pthread_cond_t  shepherd_cond;
} xk_lock;

static void* xk_thread_stub( void* arg );
static void* p_thread_stub( void* arg );

void       thread_map_dump( void );
static int thread_dump( pthread_t* pthread, xk_thread_t* thread, int* tcount );

static char* thread_status_str( xk_thread_t* thread );
static char* thread_type_str( xk_thread_t* thread );

static Map thread_map;

xk_thread_t* xk_lock_owner(void);

xk_thread_t*
xk_lock_owner(void)
{
    return xk_lock.owner;
}

xk_thread_t*
threadSelf( void )
{
    xk_thread_t* xk_thread = NULL;
    pthread_t    pthread = pthread_self();

    mapResolve(thread_map,&pthread,(void**)&xk_thread);
    xTrace2(threads,TR_NEVER,"threadSelf called with 0x%x, returning 0x%x",
            pthread, xk_thread);
    return xk_thread;
}

/*
 * threadInit is called by main, and so is guaranteed to be run by the
 * main thread. It uses that to guarantee the main thread is blocked
 * forever, so that it will never return. If it returned, _exit would
 * abort all other p-threads!
 */

void
threadInit( ThreadFunc init, void* arg )
{
    int             pthread_stacksize;
    pthread_mutex_t ever;
    pthread_cond_t  forever;
    xk_thread_t*    this_thread;

    xTrace0(threads,TR_FUNCTIONAL_TRACE,"threadInit: called");

    this_thread = (xk_thread_t*) malloc(sizeof(xk_thread_t));
    this_thread->pthread = pthread_self();
    this_thread->func    = NULL;
    this_thread->arg     = NULL;
    this_thread->type    = -1;
    this_thread->status  = XK_RELEASED;
    pthread_mutex_init( &(this_thread->mutex), pthread_mutexattr_default );
    thread_map  = mapCreate(113,sizeof(pthread_t));
    xTrace2(threads,TR_DETAILED,"threadInit: mapping 0x%x to 0x%x",
            this_thread->pthread, this_thread );
    mapBind(thread_map,&(this_thread->pthread),this_thread);

    xk_lock.owner           = NULL;
    xk_lock.access          = 1;
    xk_lock.internal_locked = 0;
    xk_lock.shepherd_locked = 0;
    pthread_mutex_init( &xk_lock.mutex,         pthread_mutexattr_default );
    pthread_cond_init(  &xk_lock.external_cond, pthread_condattr_default );
    pthread_cond_init(  &xk_lock.internal_cond, pthread_condattr_default );
    pthread_cond_init(  &xk_lock.shepherd_cond, pthread_condattr_default );

    pthread_stacksize = pthread_attr_getstacksize(pthread_attr_default);

    xTrace1(threads,TR_DETAILED,"threadInit: pthreads default stack size is %d",
	            pthread_stacksize );
    if ( pthread_stacksize < DEFAULT_STACK_SIZE ) {
	xTrace1(threads,TR_DETAILED, "Increasing stack size to %d",
	        DEFAULT_STACK_SIZE );
	if (  pthread_attr_create( &xk_thread_attr ) < 0 ) {
	    xTrace0(threads,TR_ERRORS,"threadInit couldn't create thread attr");
	    xk_thread_attr = pthread_attr_default;
	} else {
	    pthread_attr_setstacksize(&xk_thread_attr,(long)DEFAULT_STACK_SIZE);
	}
    }
    else {
	xTrace1(threads,TR_FULL_TRACE,
	        "Keeping default stack size (%d)", pthread_stacksize );
	xk_thread_attr = pthread_attr_default;
    }

    xk_thread_count=0;
    p_thread_count=1;

    xkThreadCreate( init, arg, "init func" );

    xTrace0(threads,TR_FULL_TRACE,
            "initThread: Main thread will now block forever. Bye!");
    /*
     * This is not a mistake: we really want this thread to block forever,
     * since in the Paragon Pthreads package, the return of the main
     * thread causes _exit to be called, destroying all other threads
     * and terminating the execution of the process.
     */
    pthread_mutex_init( &ever,    pthread_mutexattr_default );
    pthread_cond_init(  &forever, pthread_condattr_default );
    pthread_mutex_lock( &ever );
    pthread_cond_wait( &forever, &ever ); /* this will block! */

    xTrace0(threads,TR_ERRORS,"Main thread unlocked!");
    Kabort("*** threadInit returned - this should never have happened! ***");
}

void
xk_master_lock(void) 
{
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_master_lock: called by thread 0x%x",
            this_thread );
    pthread_mutex_lock(&xk_lock.mutex);
    xTrace1(threads,TR_MAJOR_EVENTS,"xkml 0x%x req", this_thread );
    --xk_lock.access;
    if ( xk_lock.access < 0 ) {
	xTrace1(threads,TR_FULL_TRACE,"xk_master_lock: xk_lock_access = %d",
		xk_lock.access );
	xTrace0(threads,TR_FULL_TRACE,"xk_master_lock: will wait on condition");
	this_thread->status |= XK_WAIT;
	pthread_cond_wait(&xk_lock.external_cond,&xk_lock.mutex);
	this_thread->status &= ~XK_WAIT;
	xTrace0(threads,TR_FULL_TRACE,"xk_master_lock: back from  condition");
    }
    this_thread->status |= XK_OWNER;
    xk_lock.owner = this_thread;
    xTrace1(threads,TR_FULL_TRACE,"xk_master_lock: xk_lock_access = %d",
	    xk_lock.access );
    xTrace1(threads,TR_MAJOR_EVENTS,"xkml 0x%x got", this_thread );
    pthread_mutex_unlock(&xk_lock.mutex);
    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_master_lock: thread 0x%x has the lock",
            this_thread );
}

void
xk_internal_lock(void) 
{
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_internal_lock: called by thread 0x%x",
            this_thread );
    pthread_mutex_lock(&xk_lock.mutex);
    xTrace2(threads,TR_MAJOR_EVENTS,"xkil 0x%x req(0x%x)",
	    this_thread, xk_lock.owner );
    --xk_lock.access;
    if ( xk_lock.access < 0 ) {
	xTrace1(threads,TR_FULL_TRACE,"xk_internal_lock: xk_lock_access = %d",
		xk_lock.access );
        ++xk_lock.internal_locked;
	xTrace1(threads,TR_FULL_TRACE,
	        "xk_internal_lock: will wait on condition int_locked = %d",
		xk_lock.internal_locked);
	this_thread->status |= XK_WAIT;
	pthread_cond_wait(&xk_lock.internal_cond,&xk_lock.mutex);
	this_thread->status &= ~XK_WAIT;
        --xk_lock.internal_locked;
	xTrace1(threads,TR_FULL_TRACE,
	        "xk_internal_lock: back from condition int_locked = %d",
		xk_lock.internal_locked);
    }
    this_thread->status |= XK_OWNER;
    xk_lock.owner = this_thread;
    xTrace1(threads,TR_FULL_TRACE,"xk_internal_lock: xk_lock_access = %d",
            xk_lock.access );
    xTrace1(threads,TR_MAJOR_EVENTS,"xkil 0x%x got", this_thread );
    pthread_mutex_unlock(&xk_lock.mutex);
    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_internal_lock: thread 0x%x has the lock",
            this_thread );
}

void
xk_shepherd_lock(void) 
{
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_shepherd_lock: called by thread 0x%x",
            this_thread );
    pthread_mutex_lock(&xk_lock.mutex);
    xTrace1(threads,TR_MAJOR_EVENTS,"xksl 0x%x req", this_thread );
    --xk_lock.access;
    if ( xk_lock.access < 0 ) {
	xTrace1(threads,TR_FULL_TRACE,"xk_shepherd_lock: xk_lock_access = %d",
		xk_lock.access );
        ++xk_lock.shepherd_locked;
	xTrace1(threads,TR_FULL_TRACE,
	        "xk_shepherd_lock: will wait on condition shep_locked = %d",
		xk_lock.shepherd_locked);
	this_thread->status |= XK_WAIT;
	pthread_cond_wait(&xk_lock.shepherd_cond,&xk_lock.mutex);
	this_thread->status &= ~XK_WAIT;
        --xk_lock.shepherd_locked;
	xTrace1(threads,TR_FULL_TRACE,
	        "xk_shepherd_lock: back from condition shep_locked = %d",
		xk_lock.shepherd_locked);
    }
    this_thread->status |= XK_OWNER;
    xk_lock.owner = this_thread;
    xTrace1(threads,TR_FULL_TRACE,"xk_shepherd_lock: xk_lock_access = %d",
            xk_lock.access );

    xTrace1(threads,TR_MAJOR_EVENTS,"xksl 0x%x got", this_thread );

    pthread_mutex_unlock(&xk_lock.mutex);
    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_shepherd_lock: thread 0x%x has the lock",
            this_thread );
}

void
xk_all_unlock(void)
{
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xk_all_unlock: called by thread 0x%x",
            this_thread );

    pthread_mutex_lock(&xk_lock.mutex);

    this_thread->status &= ~XK_OWNER;

    xTrace1(threads,TR_MAJOR_EVENTS,"xkul 0x%x rel", this_thread );
    assert(xk_lock.owner == this_thread);

    if ( xk_lock.access++ < 0 ) {
	xTrace1(threads,TR_FULL_TRACE,"xk_all_unlock: xk_lock.access = %d",
		xk_lock.access );
	if ( xk_lock.shepherd_locked ) {
	    xTrace1(threads,TR_FULL_TRACE,"xk_all_unlock: signal %d sheps",
		    xk_lock.shepherd_locked );
	    pthread_cond_signal(&xk_lock.shepherd_cond);
	} else if ( xk_lock.internal_locked ) {
	    xTrace1(threads,TR_FULL_TRACE,"xk_all_unlock: signal %d int",
		    xk_lock.internal_locked );
	    pthread_cond_signal(&xk_lock.internal_cond);
	} else {
	    xTrace0(threads,TR_FULL_TRACE,"xk_all_unlock: signal exts");
	    pthread_cond_signal(&xk_lock.external_cond);
	}
    }
    xTrace1(threads,TR_FULL_TRACE,"xk_all_unlock: xk_lock.access = %d",
	    xk_lock.access );

    xk_lock.owner = NULL;

    pthread_mutex_unlock(&xk_lock.mutex);

    xTrace1(threads,TR_FUNCTIONAL_TRACE,
            "xk_all_unlock: thread 0x%x released the lock", this_thread );
}

/*
 * xk_thread_stub: starts a new thread inside the x-kernel. It must first
 * lock the master mutex.
 */

static void*
xk_thread_stub( void* arg )
{
    xk_thread_t* this_thread = (xk_thread_t*) arg;
    

    xTrace1(threads,TR_DETAILED,"xk_thread_stub started as thread 0x%x",
                    this_thread );
    pthread_mutex_lock( &(this_thread->mutex) );
    this_thread->status |= XK_RELEASED;
    pthread_mutex_unlock( &(this_thread->mutex) );

    xk_internal_lock();
    xTrace3(threads,TR_FULL_TRACE,"xk_thread_stub(0x%x): calling 0x%x(0x%x)",
                    this_thread, this_thread->func, this_thread->arg );
    (this_thread->func)(this_thread->arg);
    xTrace3(threads,TR_FULL_TRACE,"xk_thread_stub(0x%x): 0x%x(0x%x) returned",
                    this_thread, this_thread->func, this_thread->arg );
    xk_all_unlock();
    xk_thread_count--;
    this_thread->status |= XK_DONE;
    xTrace2(threads,TR_DETAILED,
            "xk_thread_stub(0x%x): returns, %d xkthreads left",
            this_thread, xk_thread_count );
    mapRemoveKey(thread_map,&(this_thread->pthread));
    free((char*)this_thread);

    return NULL;     /* Just to simplify type-checking */
}

/*
 * p_thread_stub: starts a new thread outside the x-kernel.
 */

static void*
p_thread_stub( void* arg )
{
    xk_thread_t* this_thread = (xk_thread_t*) arg;

    xTrace1(threads,TR_DETAILED,"p_thread_stub started as thread 0x%x",
                    this_thread );
    pthread_mutex_lock( &(this_thread->mutex) );
    this_thread->status |= XK_RELEASED;
    pthread_mutex_unlock( &(this_thread->mutex) );

    xTrace3(threads,TR_FULL_TRACE,"p_thread_stub(0x%x): calling 0x%x(0x%x)",
                    this_thread, this_thread->func, this_thread->arg );
    (this_thread->func)(this_thread->arg);
    xTrace3(threads,TR_FULL_TRACE,"p_thread_stub(0x%x): 0x%x(0x%x) returned",
                    this_thread, this_thread->func, this_thread->arg );
    this_thread->status |= XK_DONE;
    p_thread_count--;
    xTrace2(threads,TR_DETAILED,
            "p_thread_stub(0x%x): returns, %d pthreads left",
            this_thread, p_thread_count );
    mapRemoveKey(thread_map,&(this_thread->pthread));
    free((char*)this_thread);
    return NULL;     /* Just to simplify type-checking */
}

void
xkThreadCreate( ThreadFunc func, void* arg, char* fname )
{
    xk_thread_t* new_thread;
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xkThreadCreate called by thread 0x%x",
            this_thread );

    new_thread         = (xk_thread_t*) malloc(sizeof(xk_thread_t));
    new_thread->func   = func;
    new_thread->arg    = arg;
    new_thread->type   = XK_INTERNAL;
    new_thread->status = XK_CREATED;
    pthread_mutex_init( &(new_thread->mutex), pthread_mutexattr_default );
    pthread_mutex_lock( &(new_thread->mutex) );

    if ( pthread_create(&(new_thread->pthread),pthread_attr_default,
                        xk_thread_stub,new_thread ) < 0 ){
	free((char*)new_thread);
	xError("Unable to create new thread");
	return;
    }
    xTrace2(threads,TR_DETAILED,"pThreadCreate: mapping 0x%x to 0x%x",
            new_thread->pthread, new_thread );
    mapBind(thread_map,&(new_thread->pthread),new_thread);
    xk_thread_count++;
    xTrace5(threads,TR_EVENTS,
	"xkThreadCreate: thread 0x%x created thread 0x%x, func %s 0x%x(%x)",
	this_thread, new_thread, (fname)?fname:"NULL", func, arg );
    pthread_detach( &(new_thread->pthread) );
    pthread_mutex_unlock( &(new_thread->mutex) );
    xTrace1(threads,TR_FUNCTIONAL_TRACE,"xkThreadCreate(0x%x) returning",
            this_thread );
}

void
pThreadCreate (ThreadFunc func, void* arg, char* fname )
{
    xk_thread_t* new_thread;

    xTrace2(threads,TR_FUNCTIONAL_TRACE,"pThreadCreate(0x%x,0x%x) called",
            func, arg );

    new_thread         = (xk_thread_t*) malloc(sizeof(xk_thread_t));
    new_thread->func   = func;
    new_thread->arg    = arg;
    new_thread->type   = XK_EXTERNAL;
    new_thread->status = XK_CREATED;
    pthread_mutex_init( &(new_thread->mutex), pthread_mutexattr_default );
    pthread_mutex_lock( &(new_thread->mutex) );

    if ( pthread_create(&(new_thread->pthread),pthread_attr_default,
                        p_thread_stub,new_thread)<0){
	free((char*)new_thread);
	xError("pThreadCreate: Unable to create new pthread");
	return;
    }
    xTrace2(threads,TR_DETAILED,"pThreadCreate: mapping 0x%x to 0x%x",
            new_thread->pthread, new_thread );
    mapBind(thread_map,&(new_thread->pthread),new_thread);
    p_thread_count++;
    xTrace4(threads,TR_EVENTS,
	    "pThreadCreate: thread 0x%x created %s 0x%x, %d pthreads now.",
	    threadSelf(), fname, new_thread, p_thread_count );
    pthread_detach( &(new_thread->pthread) );
    pthread_mutex_unlock( &(new_thread->mutex) );
    xTrace0(threads,TR_FUNCTIONAL_TRACE,"pThreadCreate returning" );
}

void
threadStop(void)
{
    xk_thread_t* this_thread = threadSelf();

    xk_thread_count--;
    this_thread->status |= XK_DONE;
    xTrace2(threads,TR_EVENTS,"threadStop: called by 0x%x leaving %d xkthreads",
            this_thread, xk_thread_count );
    xk_all_unlock();
    mapRemoveKey(thread_map,&(this_thread->pthread));
    free((char*)this_thread);
    pthread_exit(&xk_thread_count); /* Not really used, just make gcc happy */
}

void
threadYield(void)
{
    xk_thread_t* this_thread = threadSelf();

    xTrace1(threads,TR_EVENTS,"threadYield: thread 0x%x yielding now",
            this_thread);
    this_thread->status |= XK_YIELDED;
    xk_all_unlock();
	xTrace1(threads,TR_MAJOR_EVENTS,"threadYield: thread 0x%x yielding now",
		this_thread);
	pthread_yield();
    this_thread->status &= ~XK_YIELDED;
    xk_internal_lock();
    xTrace1(threads,TR_EVENTS,"threadYield: thread 0x%x returning from yield",
            this_thread);
}

void
thread_map_dump( void )
{
    int thread_count = 0;
    /*
     * prints all information available about the state of kernel threads.
     */
     xTrace0(threads,TR_ALWAYS,"thread_map_dump: all threads will be printed");
     mapForEach( thread_map, (MapForEachFun) thread_dump, &thread_count);
     xTrace1(threads,TR_ALWAYS,"thread_map_dump: all threads (%d) were printed",
             thread_count);
}

static int
thread_dump( pthread_t* pthread, xk_thread_t* thread, int* tcount )
{
    char buf[64];

    *tcount += 1;
    sprintf(buf,"0x%x(0x%x",thread->func,thread->arg);
    assert( *pthread == thread->pthread );
    xTrace6(threads,TR_ALWAYS,"thread 0x%x [%02d]: pthread 0x%x, func %s, "
                              "type %s, status %s",
                              thread, *tcount, thread->pthread, buf,
                              thread_type_str( thread ),
                              thread_status_str( thread ) );
    return MFE_CONTINUE;
}

static char*
thread_status_str( xk_thread_t* thread )
{
    static char buffer[128];
    int  status = thread->status;

#define CHECK_STATUS( _ST ) if (status&PASTE(XK_,_ST)) strcat(buffer,#_ST " ");

    strcpy(buffer,"{ ");
    CHECK_STATUS( CREATED );
    CHECK_STATUS( RELEASED );
    CHECK_STATUS( WAIT );
    CHECK_STATUS( OWNER ); 
    CHECK_STATUS( SEM_WAIT );
    CHECK_STATUS( SEM_OWN );
    CHECK_STATUS( DELAYED );
    CHECK_STATUS( YIELDED );
    CHECK_STATUS( DONE );
    strcat(buffer,"}");

    return buffer;
}

static char*
thread_type_str( xk_thread_t* thread )
{
    switch( thread->type ) {
        case XK_EXTERNAL: return "EXTERNAL";
        case XK_INTERNAL: return "INTERNAL";
        case XK_EVENT:    return "EVENT";
        case XK_SHEPHERD: return "SHEPHERD";
        default:       return "UNKNOWN";
    }
    return NULL;
}