/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * This source file is part of SableVM.                            *
 *                                                                 *
 * See the file "LICENSE" for the copyright information and for    *
 * the terms and conditions for copying, distribution and          *
 * modification of this source file.                               *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/* Thread specific data */
static pthread_key_t _svmv_env_key;	/* key to access thread specific environment data */

/* stack_offset */
static const size_t _svmv_stack_offset =
  (sizeof (_svmt_stack_frame) +
   (SVM_ALIGNMENT - 1)) & ~((size_t) (SVM_ALIGNMENT - 1));

/*
----------------------------------------------------------------------
_svmf_thread_init
----------------------------------------------------------------------
*/

svm_static jint
_svmf_thread_init (void)
{
  /* create key for thread specific data */
  if (pthread_key_create (&_svmv_env_key, NULL) != 0)
    {
      goto error;
    }

  return JNI_OK;

error:
  return JNI_ERR;
}

/*
----------------------------------------------------------------------
_svmf_get_current_env
----------------------------------------------------------------------
*/

static _svmt_JNIEnv *
_svmf_get_current_env (void)
{
  return (_svmt_JNIEnv *) pthread_getspecific (_svmv_env_key);
}

/*
----------------------------------------------------------------------
_svmf_set_current_env
----------------------------------------------------------------------
*/

static void
_svmf_set_current_env (_svmt_JNIEnv *env)
{
#ifndef NDEBUG
  int error =
#endif
    pthread_setspecific (_svmv_env_key, env);

  assert (!error);
}

/*
----------------------------------------------------------------------
_svmf_stack_init_defaults
----------------------------------------------------------------------
*/

svm_static void
_svmf_stack_init_defaults (_svmt_JavaVM *vm)
{
  vm->stack_min_size = SVM_STACK_DEFAULT_MIN_SIZE;
  vm->stack_max_size = SVM_STACK_DEFAULT_MAX_SIZE;
  vm->stack_allocation_increment = SVM_STACK_DEFAULT_ALLOCATION_INCREMENT;
}

/*
----------------------------------------------------------------------
_svmf_stack_init
----------------------------------------------------------------------
*/

svm_static jint
_svmf_stack_init (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;
  size_t lrefs_offset = _svmf_aligned_size_t (sizeof (_svmt_stack_frame));
  jint lrefs_count = SVM_FRAME_NATIVE_REFS_MIN;
  size_t lrefs_size =
    _svmf_aligned_size_t ((lrefs_count +
			   2) * sizeof (_svmt_stack_native_reference));
  size_t frame_size = lrefs_offset + lrefs_size;
  size_t alloc_size;

  if (frame_size <= vm->stack_min_size)
    {
      alloc_size = vm->stack_min_size;
    }
  else if (vm->stack_max_size == 0 || frame_size <= vm->stack_max_size)
    {
      alloc_size = vm->stack_min_size + _svmf_aligned_to_increment
	(frame_size - vm->stack_min_size, vm->stack_allocation_increment);
    }
  else
    {
      _svmf_error_OutOfMemoryError (env);
      return JNI_ERR;
    }

  env->stack.start = _svmf_malloc (alloc_size);

  if (env->stack.start == NULL)
    {
      _svmf_error_OutOfMemoryError (env);
      return JNI_ERR;
    }

  env->stack.current_frame = env->stack.start;
  env->stack.end = ((char *) env->stack.start) + alloc_size;

  env->stack.current_frame->previous_offset = 0;
  env->stack.current_frame->end_offset = frame_size;
  env->stack.current_frame->method = &vm->stack_bottom_method;
  env->stack.current_frame->stack_trace_element = NULL;
  env->stack.current_frame->lock_count = 0;
  env->stack.current_frame->this = NULL;
  env->stack.current_frame->pc = vm->stack_bottom_method.frame_info->code;
  env->stack.current_frame->stack_size = 0;

  {
    _svmt_stack_native_reference *lrefs =
      (_svmt_stack_native_reference *) (void
					*) (((char *) env->stack.
					     current_frame) +
					    env->stack.current_frame->
					    end_offset);
    jint i;

    lrefs[-1].jint = lrefs_count;
    lrefs[-2].size_t = lrefs_size;

    lrefs =
      (_svmt_stack_native_reference *) (void *) (((char *) lrefs) -
						 lrefs_size);

    for (i = 0; i < lrefs_count; i++)
      {
	if (_svmm_new_native_local (env, lrefs[i].jobject) != JNI_OK)
	  {
	    return JNI_ERR;
	  }
      }
  }

  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_ensure_stack_capacity
----------------------------------------------------------------------
*/

svm_static jint
_svmf_ensure_stack_capacity (_svmt_JNIEnv *env, size_t frame_size)
{
  void *current_frame = env->stack.current_frame;
  void *current_frame_end =
    ((char *) current_frame) + env->stack.current_frame->end_offset;
  void *stack_end = env->stack.end;
  size_t available = ((char *) stack_end) - ((char *) current_frame_end);

  assert (stack_end >= current_frame_end);

#ifdef STATISTICS

  {
    _svmt_JavaVM *vm = env->vm;
    void *stack_start = env->stack.start;
    _svmt_stack_frame *frame = (_svmt_stack_frame *) current_frame;

    size_t current_stack_size = ((char *) frame) - ((char *) stack_start);

    if (current_stack_size > vm->max_stack_size)
      {
	_svmt_method_info *method = env->stack.current_frame->method;

	vm->max_stack_size = current_stack_size;
	vm->stack_local_count = 0;
	vm->stack_local_split_count = 0;

	while (method != &vm->stack_bottom_method)
	  {
	    if (!_svmf_is_set_flag (method->access_flags, SVM_ACC_INTERNAL))
	      {
		vm->stack_local_count += method->frame_info->local_count;
		vm->stack_local_split_count +=
		  method->frame_info->local_split_count;
	      }

	    frame = (_svmt_stack_frame *)
	      (((char *) frame) - frame->previous_offset);
	    method = frame->method;
	  }
      }
  }

#endif

  if (frame_size > available)
    {
      _svmt_JavaVM *vm = env->vm;
      size_t stack_increment = vm->stack_allocation_increment;

      if (stack_increment == 0)
	{
	  _svmf_error_OutOfMemoryError (env);
	  return JNI_ERR;
	}
      else
	{
	  size_t min_growth = frame_size - available;
	  void *stack_start = env->stack.start;
	  size_t current_frame_offset =
	    ((char *) current_frame) - ((char *) stack_start);

	  /* growth sould be a multiple of stack_increment large
	     enough to hold min_growth */
	  size_t growth =
	    _svmf_aligned_to_increment (min_growth, stack_increment);
	  size_t current_size = ((char *) stack_end) - ((char *) stack_start);
	  size_t new_size = current_size + growth;
	  void *new_stack;

	  /* detect overflows */
	  if ((vm->stack_max_size != 0 && new_size > vm->stack_max_size) ||
	      new_size <= current_size)
	    {
	      _svmf_error_OutOfMemoryError (env);
	      return JNI_ERR;
	    }

	  new_stack = _svmf_realloc (stack_start, new_size);

	  /* out of memory */
	  if (new_stack == NULL)
	    {
	      _svmf_error_OutOfMemoryError (env);
	      return JNI_ERR;
	    }

	  env->stack.start = new_stack;
	  env->stack.end = ((char *) new_stack) + new_size;
	  env->stack.current_frame =
	    (_svmt_stack_frame *) (void *) (((char *) new_stack) +
					    current_frame_offset);
	}
    }

  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_halt_if_requested
----------------------------------------------------------------------
*/

/* IMPORTANT: The calling thread should hold the lock on
   vm->global_mutex when calling this function. */

svm_static void
_svmf_halt_if_requested (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;

  assert (env->thread_status != SVM_THREAD_STATUS_HALTED);
  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED);
  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);

  /* is halting requested by another thread? */
  while (env->thread_status == SVM_THREAD_STATUS_HALT_REQUESTED)
    {
      env->thread_status = SVM_THREAD_STATUS_HALTED;

      /* there must be a reason for halting */
      assert (env->status_flags != SVM_THREAD_STATUS_FLAGS_NONE);

      /* are we halting for (among other things) stopping the world? */
      if (_svmf_is_set_flag
	  (env->status_flags, SVM_THREAD_STATUS_FLAGS_STOP_THE_WORLD))
	{
	  assert (vm->stop_the_world.pending_halt_thread_count > 0);

	  vm->stop_the_world.pending_halt_thread_count--;

	  /* is there any other thread pending halt? */
	  if (vm->stop_the_world.pending_halt_thread_count == 0)
	    {
	      /* no, so signal requesting thread */
	      _svmm_cond_signal (vm->stop_the_world.requesting_thread_cond);
	    }
	}

      /* are we halting for (among other things) suspending this thread? */
      if (_svmf_is_set_flag
	  (env->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND))
	{
	  _svmt_JNIEnv *current;

	  /* signal all threads that requested the suspension of this
	     thread. suspender_list could be NULL if the thread has 
	     suspended itself and is the only that call suspend_thread. */
	  current = env->suspension.suspender_list;

	  while (current != NULL)
	    {
	      _svmm_cond_signal (current->suspension.cond);
	      current = current->suspension.suspender_list_next;
	    }
	}

      /* time to halt the thread */
      do
	{
	  _svmm_cond_wait (env->wakeup_cond, vm->global_mutex);
	}
      while (env->thread_status == SVM_THREAD_STATUS_HALTED);
    }

  assert (env->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);
}

/*
----------------------------------------------------------------------
_svmf_periodic_check
----------------------------------------------------------------------
*/

inline svm_static void
_svmf_periodic_check (_svmt_JNIEnv *env)
{
#ifndef NDEBUG
  jint status = env->thread_status;
  assert (status == SVM_THREAD_STATUS_RUNNING_JAVA ||
	  status == SVM_THREAD_STATUS_HALT_REQUESTED);
#endif

  /* is stopping the world requested by another thread? */
  if (env->thread_status == SVM_THREAD_STATUS_HALT_REQUESTED)
    {
      _svmt_JavaVM *vm = env->vm;

      _svmm_mutex_lock (vm->global_mutex);

      _svmf_halt_if_requested (env);

      _svmm_mutex_unlock ();
    }
}

/*   
---------------------------------------------------------------------   
_svmf_is_suspended
---------------------------------------------------------------------   
*/

/* IMPORTANT: The calling thread should hold the lock on
   vm->global_mutex when calling this function. */

static jboolean
_svmf_is_suspended (_svmt_JNIEnv *target)
{
  jboolean result = JNI_FALSE;

  if (((target->thread_status == SVM_THREAD_STATUS_HALTED) ||
       (target->thread_status ==
	SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED))
      && _svmf_is_set_flag (target->status_flags,
			    SVM_THREAD_STATUS_FLAGS_SUSPEND))
    {
      result = JNI_TRUE;
    }

  return result;
}

/*   
---------------------------------------------------------------------   
_svmf_suspend_thread
---------------------------------------------------------------------   
*/

/* IMPORTANT: The calling thread should hold the lock on
   vm->global_mutex when calling this function.  Once the thread is
   halted, the calling thread should release the lock until just
   prior the call to resume_thread. 

   returns: 
     JNI_FALSE: the thread was already suspended
     JNI_TRUE: the thread has been suspended by this call
*/

static jboolean
_svmf_suspend_thread (_svmt_JNIEnv *env, _svmt_JNIEnv *target)
{
  jboolean result, succeeded;
  _svmt_JavaVM *vm = env->vm;

  /* halt if requested */
  _svmf_halt_if_requested (env);

  /* suspend the calling thread */
  if (target == env)
    {
      assert (target->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);

      target->thread_status = SVM_THREAD_STATUS_HALTED;

      _svmm_set_flag (target->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND);

      assert (env->suspension.suspender_list_next == NULL);

      /* halt the thread */
      do
	{
	  _svmm_cond_wait (env->wakeup_cond, vm->global_mutex);
	}
      while (_svmf_is_suspended (target));
    }
  /* suspend another thread */
  else
    {
      do
	{
	  switch (target->thread_status)
	    {
	    case SVM_THREAD_STATUS_RUNNING_JAVA:
	      {
		succeeded = _svmm_compare_and_swap
		  (target->thread_status,
		   SVM_THREAD_STATUS_RUNNING_JAVA,
		   SVM_THREAD_STATUS_HALT_REQUESTED);
	      }
	      break;

	    case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED:
	      {
		succeeded = _svmm_compare_and_swap
		  (target->thread_status,
		   SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED,
		   SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);
	      }
	      break;

	    case SVM_THREAD_STATUS_HALT_REQUESTED:
	    case SVM_THREAD_STATUS_HALTED:
	    case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED:
	      {
		succeeded = JNI_TRUE;
	      }
	      break;

	    default:
	      {
		succeeded = JNI_FALSE;	/* to keep compiler happy */
		_svmm_fatal_error ("impossible control flow");
	      }
	      break;
	    }
	}
      while (!succeeded);

      if (_svmf_is_suspended (target))
	{
	  /* The thread was already halted for suspension */
	  result = JNI_FALSE;
	  goto end;
	}

      _svmm_set_flag (target->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND);

      if (target->thread_status == SVM_THREAD_STATUS_HALTED ||
	  target->thread_status ==
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED)
	{
	  /* The thread was already halted for some reason or not
	     running java */
	  result = JNI_TRUE;
	  goto end;
	}

      assert (target->thread_status == SVM_THREAD_STATUS_HALT_REQUESTED);
      assert (env->suspension.suspender_list_next == NULL);

      env->suspension.suspender_list_next = target->suspension.suspender_list;
      target->suspension.suspender_list = env;

      /* wait until target is halted */
      do
	{
	  _svmm_cond_wait (env->suspension.cond, vm->global_mutex);
	}
      while (target->thread_status == SVM_THREAD_STATUS_HALT_REQUESTED);

      /* once the target thread is halted (or it has been resumed by
         another thread) */
      {
	_svmt_JNIEnv *previous = NULL;
	_svmt_JNIEnv *current = target->suspension.suspender_list;

	/* remove this thread from the suspender_list */
	while (current != env)
	  {
	    previous = current;
	    current = current->suspension.suspender_list_next;

	    assert (current != NULL);
	  }

	assert (current == env);

	if (previous != NULL)
	  {
	    previous->suspension.suspender_list_next =
	      env->suspension.suspender_list_next;
	  }
	else
	  {
	    target->suspension.suspender_list =
	      env->suspension.suspender_list_next;
	  }

	env->suspension.suspender_list_next = NULL;
      }
    }

  result = JNI_TRUE;

end:

  return result;
}


/*   
---------------------------------------------------------------------   
_svmf_resume_thread
---------------------------------------------------------------------   
*/

/* IMPORTANT: The calling thread should hold the lock on
   vm->global_mutex when calling this function.

   returns: 
     JNI_FALSE: the thread was already resumed
     JNI_TRUE: the thread has been resumed by this call
*/

static jboolean
_svmf_resume_thread (_svmt_JNIEnv *env, _svmt_JNIEnv *target)
{
  jboolean result;

  /* halt if requested */
  _svmf_halt_if_requested (env);

  assert (env->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);

  if (target == env)
    {
      /* if the current thread calls this function on itself, it cant
         be sleeping */
      assert (!_svmf_is_set_flag
	      (target->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND));

      result = JNI_FALSE;
      goto end;
    }

  if (!_svmf_is_set_flag
      (target->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND))
    {
      /* target thread is not suspended */

      result = JNI_FALSE;
      goto end;
    }

  /* unsuspend target thread */
  _svmm_clear_flag (target->status_flags, SVM_THREAD_STATUS_FLAGS_SUSPEND);

  if (target->thread_status == SVM_THREAD_STATUS_HALT_REQUESTED)
    {
      _svmt_JNIEnv *current;

      /* signal all threads that have requested the suspension of the
         target thread. */
      current = target->suspension.suspender_list;

      while (current != NULL)
	{
	  _svmm_cond_signal (current->suspension.cond);
	  current = current->suspension.suspender_list_next;
	}
    }

  if (target->status_flags != SVM_THREAD_STATUS_FLAGS_NONE)
    {
      /* target must remain halted for some other reason */

      result = JNI_TRUE;
      goto end;
    }

  switch (target->thread_status)
    {
    case SVM_THREAD_STATUS_HALTED:
      {
	target->thread_status = SVM_THREAD_STATUS_RUNNING_JAVA;
	_svmm_cond_signal (target->wakeup_cond);
      }
      break;

    case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED:
      {
	target->thread_status =
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED;
	_svmm_cond_signal (target->wakeup_cond);
      }
      break;

    case SVM_THREAD_STATUS_HALT_REQUESTED:
      {
	target->thread_status = SVM_THREAD_STATUS_RUNNING_JAVA;
      }
      break;

    case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED:
    case SVM_THREAD_STATUS_RUNNING_JAVA:
    default:
      {
	_svmm_fatal_error ("impossible control flow");
      }
      break;
    }

  result = JNI_TRUE;

end:
  return result;
}

/*
----------------------------------------------------------------------
_svmf_stop_the_world
----------------------------------------------------------------------
*/

/* IMPORTANT: The calling thread should hold the lock on
   vm->global_mutex when calling this function.  Once the world is
   stopped, the calling thread should release the lock until just
   prior the call to resume_the_world. */

svm_static void
_svmf_stop_the_world (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;
  jint i;

  /* if another thread beat us to it, halt */
  _svmf_halt_if_requested (env);

  /* ok, now we can proceed */
  assert (env->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);
  assert (vm->stop_the_world.pending_halt_thread_count == 0);

  for (i = 0; i < 2; i++)
    {
      _svmt_JNIEnv *current;

      /* visit all threads */
      for (current = ((i == 0) ? vm->threads.user : vm->threads.system);
	   current != NULL; current = current->next)
	{
	  jboolean succeeded;

	  /* skip the running thread */
	  if (current == env)
	    {
	      continue;
	    }

	  /* request halt */
	  do
	    {
	      switch (current->thread_status)
		{
		case SVM_THREAD_STATUS_RUNNING_JAVA:
		  {
		    succeeded = _svmm_compare_and_swap
		      (current->thread_status, SVM_THREAD_STATUS_RUNNING_JAVA,
		       SVM_THREAD_STATUS_HALT_REQUESTED);

		    if (succeeded)
		      {
			vm->stop_the_world.pending_halt_thread_count++;
		      }
		  }
		  break;

		case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED:
		  {
		    succeeded = _svmm_compare_and_swap
		      (current->thread_status,
		       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED,
		       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);
		  }
		  break;

		case SVM_THREAD_STATUS_HALT_REQUESTED:
		case SVM_THREAD_STATUS_HALTED:
		case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED:
		  {
		    succeeded = JNI_TRUE;
		  }
		  break;

		default:
		  {
		    succeeded = JNI_FALSE;	/* to keep compiler happy */
		    _svmm_fatal_error ("impossible control flow");
		  }
		  break;
		}
	    }
	  while (!succeeded);

	  _svmm_set_flag (current->status_flags,
			  SVM_THREAD_STATUS_FLAGS_STOP_THE_WORLD);
	}
    }

  /* wait for other threads to halt */
  while (vm->stop_the_world.pending_halt_thread_count != 0)
    {
      _svmm_cond_wait (vm->stop_the_world.requesting_thread_cond,
		       vm->global_mutex);
    }

  /* the world is stopped! we can resume... */
  return;
}

/*
----------------------------------------------------------------------
_svmf_resume_the_world
----------------------------------------------------------------------
*/

/* IMPORTANT: The calling thread should acquire the lock on
   vm->global_mutex before calling this function.  Of course, this
   function should not be invoked unless the world has been already
   stopped by the calling thread. */

svm_static void
_svmf_resume_the_world (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;
  jint i;

  assert (env->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);
  assert (vm->stop_the_world.pending_halt_thread_count == 0);

  for (i = 0; i < 2; i++)
    {
      _svmt_JNIEnv *current;

      /* visit all threads */
      for (current = ((i == 0) ? vm->threads.user : vm->threads.system);
	   current != NULL; current = current->next)
	{
	  /* skip the running thread */
	  if (current == env)
	    {
	      continue;
	    }


	  assert (_svmf_is_set_flag
		  (current->status_flags,
		   SVM_THREAD_STATUS_FLAGS_STOP_THE_WORLD));

	  /* clear stop the world status */
	  _svmm_clear_flag (current->status_flags,
			    SVM_THREAD_STATUS_FLAGS_STOP_THE_WORLD);

	  if (current->status_flags == SVM_THREAD_STATUS_FLAGS_NONE)
	    {
	      switch (current->thread_status)
		{
		case SVM_THREAD_STATUS_HALTED:
		  {
		    current->thread_status = SVM_THREAD_STATUS_RUNNING_JAVA;
		    _svmm_cond_signal (current->wakeup_cond);
		  }
		  break;

		case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED:
		  {
		    current->thread_status =
		      SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED;
		    _svmm_cond_signal (current->wakeup_cond);
		  }
		  break;

		case SVM_THREAD_STATUS_HALT_REQUESTED:
		case SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED:
		case SVM_THREAD_STATUS_RUNNING_JAVA:
		default:
		  {
		    _svmm_fatal_error ("impossible control flow");
		  }
		  break;
		}
	    }
	}
    }

  /* the world will resume execution as soon as this thread releases
     the global mutex... */
  return;
}

#ifndef NDEBUG
/*
----------------------------------------------------------------------
_svmh_assert_not_encloser
----------------------------------------------------------------------
*/

inline svm_static void
_svmh_assert_not_encloser (_svmt_JNIEnv *env,
			   _svmt_object_instance *instance,
			   const char *file, int line)
{
  _svmt_word lockword = instance->lockword;
  if (!_svmf_lockword_is_thin (lockword))
    {
      _svmt_word body_index = _svmf_lockword_get_fatlock_index (lockword);
      _svmt_JNIEnv *main_body = env->vm->threads.array[body_index];
      if (main_body == env->locking.encloses)
	{
	  _svmf_dump_locking_structures (env);
	  printf ("THREAD: %i/%i, FILE: %s:%i\n", env->thread.id, main_body,
		  file, line);
	  fflush (NULL);
	  _svmm_fatal_error ("we were not supposed to be the encloser");
	}
    }
}
#endif

/*
----------------------------------------------------------------------
_svmf_dump_locking_structures
----------------------------------------------------------------------
*/

#ifdef NDEBUG
#define _svmf_dump_locking_structures(env)

#else

svm_static void
_svmf_dump_locking_structures (_svmt_JNIEnv *env)
{
  int i;
  _svmt_JavaVM *vm = env->vm;

  printf("DUMP FROM THREAD: %i\n", env->thread.id);

  for (i = 1; i < vm->threads.next_thread_id; i++)
    {
      _svmt_JNIEnv *current = vm->threads.array[i];

      assert (current);

      printf ("------------- THREAD %i LOCKING DATA -------------\n",
	      current->thread.id);

      /* CONTENTION */
      printf ("CONTENTION: flag: %i, wakeup_reason %i, ",
	      current->contention.flag, current->contention.wakeup_reason);

      if (current->contention.semi_attached_next)
	{
	  printf ("semi_attached_next %i\n",
		  current->contention.semi_attached_next->thread.id);
	}
      else
	{
	  printf ("semi_attached_next NULL\n");
	}

      if (current->contention.semi_attached_list)
	{
	  _svmt_JNIEnv *curr_semi;
	  printf ("semi_attached_list: \n");

	  /* go thru the list of semi attached locks */
	  for (curr_semi = current->contention.semi_attached_list;
	       curr_semi;
	       curr_semi = curr_semi->contention.semi_attached_next)
	    {
	      printf ("  FL:%i inst:%p\n", curr_semi->thread.id,
		      *curr_semi->body.jobject);
	    }
	  printf ("\n");
	}
      else
	{
	  printf ("semi_attached_list: NULL\n");
	}

      /* LOCKING */
      printf ("LOCKING: interrupted:%i, waiting_on:%p, ",
	      current->locking.interrupted, *current->locking.waiting_on);
      if (_svmm_mutex_trylock (current->locking.thread_mutex) != EBUSY)
	{
	  _svmm_mutex_unlock_after_try (current->locking.thread_mutex);
	  printf ("thread_mutex: UNlocked, ");
	}
      else
	{
	  printf ("thread_mutex: locked, ");
	}

      printf ("thrmx@%p, ", &current->locking.thread_mutex);
      printf ("thrcnd@%p, ", &current->locking.thread_cond);

      if (current->locking.prev)
	{
	  printf ("prev %i, ", current->locking.prev->thread.id);
	}
      else
	{
	  printf ("prev NULL, ");
	}

      if (current->locking.next)
	{
	  printf ("next %i\n", current->locking.next->thread.id);
	}
      else
	{
	  printf ("next NULL\n");
	}

      /* BODY */
      printf ("BODY: id: %i, ", current->thread.id);

      printf ("encloser: %i, ", current->body.encloser->thread.id);

      if (current->body.owner)
	{
	  printf ("owner: %i, ", current->body.owner->thread.id);
	}
      else
	{
	  printf ("owner: NULL, ");
	}
      printf ("reccnt: %i, instance: %p, ",
	      current->body.recursive_count, *current->body.jobject);
      if (_svmm_mutex_trylock (current->body.mutex) != EBUSY)
	{
	  _svmm_mutex_unlock_after_try (current->body.mutex);
	  printf ("body.mutex: UNlocked, ");
	}
      else
	{
	  printf ("body.mutex: locked, ");
	}
      printf ("flmx@%p, ", &current->body.mutex);
      printf ("wtcnd@%p, ", &current->body.wait_cond);

      {
	/* WAIT/BLOCKED LISTS in full */
	_svmt_JNIEnv *curr_elem;

	printf ("WAIT: ");
	for (curr_elem = current->body.list_wait_head;
	     curr_elem; curr_elem = curr_elem->locking.next)
	  {
	    printf ("%i, ", curr_elem->thread.id);
	  }
	printf (" TAIL: %p", current->body.list_wait_tail);

	printf (" BLOCKED: ");
	for (curr_elem = current->body.list_blocked_head;
	     curr_elem; curr_elem = curr_elem->locking.next)
	  {
	    printf ("%i, ", curr_elem->thread.id);
	  }
	printf (" TAIL: %p", current->body.list_blocked_tail);
      }

      {
	int j;
	printf ("\nLAST MUTEX OPS:\n");
	j = (current->mtrace.lastopno + 1) % MX_TRACE_NO;
	do
	  {
	    char op[2];
	    op[1] = 0;
	    switch (current->mtrace.lockop[j])
	      {
	      case 0:
		op[0] = 'u';
		break;
	      case 1:
		op[0] = 'l';
		break;
	      case 2:
		op[0] = 'w';
		break;
	      case 3:
		op[0] = 's';
		break;
	      case 'g':	/* GC */
		/* monitor eXit, eNter, Wait, Signal-notify/all */
	      case 'X':
	      case 'N':
	      case 'W':
	      case 'S':
		op[0] = current->mtrace.lockop[j];
		break;
	      default:
		op[0] = current->mtrace.lockop[j];
		break;
	      }
	    printf ("%i:%p %s:%i,l:%s, d:%i, inst:%p\n",
		    current->mtrace.globalno[j], current->mtrace.ptr[j],
		    current->mtrace.file[j], current->mtrace.line[j], op,
		    current->mtrace.opdone[j], current->mtrace.instance[j]);
	    j = (j + 1) % MX_TRACE_NO;
	  }
	while (j != (current->mtrace.lastopno + 1) % MX_TRACE_NO);
      }

      printf ("\n");
      fflush (NULL);
    }
#if 0
  if (!vm->mtrace_done)
    {
      int j = 0;
      vm->mtrace_done = 1;
      printf ("\nREGISTERED GLOBAL MUTEX OPS:\n");
#if 0
      if (env->vm->mutex_lastopno > MX_GLOBAL_TRACE_NO)
	j = MX_GLOBAL_TRACE_NO;
      else
	j = env->vm->mutex_lastopno;

      j -= 10000;		/* print the last 10000 entries */

      if (j < 0)
	j = 0;
#endif
      for (; j <= MX_GLOBAL_TRACE_NO; j++)
	{
	  char op[2];
	  op[1] = 0;
	  switch (vm->mtrace.lockop[j])
	    {
	    case 0:
	      op[0] = 'u';
	      break;
	    case 1:
	      op[0] = 'l';
	      break;
	    case 2:
	      op[0] = 'w';
	      break;
	    case 3:
	      op[0] = 's';
	      break;
	    case 'g':		/* GC */
	      /* monitor eXit, eNter, Wait, Signal-notify/all */
	    case 'X':
	    case 'N':
	    case 'W':
	    case 'S':
	      op[0] = vm->mtrace.lockop[j];
	      break;
	    default:
	      op[0] = '?';
	      break;
	    }
	  printf ("%i[%i] %s:%i,l:%s, d:%i, ptr:%p, inst:%p\n", j,
		  vm->mtrace.thread_id[j], vm->mtrace.file[j],
		  vm->mtrace.line[j], op, vm->mtrace.opdone[j],
		  vm->mtrace.ptr[j], vm->mtrace.instance[j]);
	}
    }
#endif

/* composite dump */
  {
    int glob_min = -1, glob_max = -1, glob, next_glob;

    printf ("========== COMPOSITE DUMP ==============\n");
    /* find the lowest entry no */
    for (i = 1; i < vm->threads.next_thread_id; i++)
      {
	_svmt_JNIEnv *current = vm->threads.array[i];
	int thr_min_no = (current->mtrace.lastopno + 1) % MX_TRACE_NO;
	int thr_max_no = (current->mtrace.lastopno) % MX_TRACE_NO;

	if ((glob_min == -1) ||
	    (current->mtrace.globalno[thr_min_no] < glob_min))
	  {
	    if (current->mtrace.globalno[thr_min_no] > 0)
	      {
		glob_min = current->mtrace.globalno[thr_min_no];
	      }
	  }
	if ((glob_max == -1) ||
	    (current->mtrace.globalno[thr_max_no] > glob_max))
	  {
	    glob_max = current->mtrace.globalno[thr_max_no];
	  }
      }
    printf ("glob_min: %i, glob_max %i\n", glob_min, glob_max);

    sleep (1);			/* wait until all non-blocked threads "ran away", so
				 * that we didn't see them in the dump. */
    if (glob_min <= 0)
      {
	glob_min = 1;
      }
    /* print entries in a composite fashion. It's suboptimal, I know. */
    for (glob = glob_min; glob <= glob_max; glob++)
      {
	next_glob = glob_max + 1;
	for (i = 1; i < vm->threads.next_thread_id; i++)
	  {
	    _svmt_JNIEnv *current = vm->threads.array[i];
	    int k;
	    for (k = 0; k < MX_TRACE_NO; k++)
	      {
		if ((current->mtrace.globalno[k] > glob) &&
		    (current->mtrace.globalno[k] < next_glob))
		  {
		    next_glob = current->mtrace.globalno[k];
		  }
		/* print this one */
		if (current->mtrace.globalno[k] == glob)
		  {
		    char op[2] = "!";
		    switch (current->mtrace.lockop[k])
		      {
		      case 0:
			op[0] = 'u';
			break;
		      case 1:
			op[0] = 'l';
			break;
		      case 2:
			op[0] = 'w';
			break;
		      case 3:
			op[0] = 's';
			break;
		      case 'g':	/* GC */
			/* monitor eXit, eNter, Wait, Signal-notify/all */
		      case 'X':
		      case 'N':
		      case 'W':
		      case 'S':
			op[0] = current->mtrace.lockop[k];
			break;
		      default:
			op[0] = current->mtrace.lockop[k];
			break;
		      }
		    printf ("T(%i) %i:%p %s:%i,l:%s, d:%i, inst:%p\n",
			    current->thread.id, current->mtrace.globalno[k],
			    current->mtrace.ptr[k], current->mtrace.file[k],
			    current->mtrace.line[k], op,
			    current->mtrace.opdone[k],
			    current->mtrace.instance[k]);
		  }
	      }
	  }
	glob = next_glob - 1;
      }
    fflush (NULL);
  }

  fflush (NULL);
  while (1)  { }
}
#endif

/*
----------------------------------------------------------------------
_svmf_lock_thin_increment_overflow
----------------------------------------------------------------------
*/

svm_static inline jint
_svmf_lock_thin_increment_overflow (_svmt_JNIEnv *env,
				    _svmt_object_instance *instance,
				    _svmt_word lockword,
				    jint new_recursive_count)
{

  if (new_recursive_count > SVM_THINLOCK_MAX_RECURSIVE_COUNT)
    {
      _svmt_overflown_counter *last_overflown_counter = NULL;
      _svmt_overflown_counter *overflown_counter =
	env->overflown_counter_list_head;
      while (overflown_counter != NULL)
	{
	  if (*overflown_counter->jobject == instance)
	    {
	      overflown_counter->count++;
	      if (last_overflown_counter != NULL)
		{		/* Make it first */
		  last_overflown_counter->next = overflown_counter->next;
		  overflown_counter->next = env->overflown_counter_list_head;
		  env->overflown_counter_list_head = overflown_counter;
		}
	      return JNI_OK;
	    }
	  overflown_counter = overflown_counter->next;
	}
      _svmm_fatal_error ("impossible control flow");
    }

  /* Otherwise - the counter has just overflown.  This is the least
   * probable case, so it's as the last one here. */
  if (new_recursive_count == SVM_THINLOCK_MAX_RECURSIVE_COUNT)
    {
      _svmt_overflown_counter *overflown_counter =
	env->overflown_counter_free_list;
      /* Get us a counter first */
      if (overflown_counter == NULL)
	{
	  if (_svmm_gzalloc_overflown_counter
	      (env, overflown_counter) != JNI_OK)
	    {
	      return JNI_ERR;	/* OOM */
	    }
	  if (_svmm_new_native_local
	      (env, overflown_counter->jobject) != JNI_OK)
	    {
	      _svmm_gzfree_overflown_counter (overflown_counter);
	      return JNI_ERR;	/* OOM */
	    }
	}
      else
	{			/* We took it away from the free list */
	  env->overflown_counter_free_list = overflown_counter->next;
	}

      instance->lockword = _svmf_lockword_thinlock
	(env->thread.thinlock_id, new_recursive_count,
	 _svmf_lockword_get_extra_bits (lockword));

      *overflown_counter->jobject = instance;
      overflown_counter->count = new_recursive_count;
      overflown_counter->next = env->overflown_counter_list_head;
      env->overflown_counter_list_head = overflown_counter;
    }
  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_lock_thin_decrement_overflow
----------------------------------------------------------------------
*/

svm_static inline void
_svmf_lock_thin_decrement_overflow (_svmt_JNIEnv *env,
				    _svmt_object_instance *instance,
				    _svmt_word lockword)
{

  /* The counter is oveflown, look for the struct. */
  _svmt_overflown_counter *last_overflown_counter = NULL;
  _svmt_overflown_counter *overflown_counter =
    env->overflown_counter_list_head;
  while (overflown_counter != NULL)
    {
      if (*overflown_counter->jobject == instance)
	{
	  jint recursive_count = --overflown_counter->count;

	  if (recursive_count == SVM_THINLOCK_MAX_RECURSIVE_COUNT - 1)
	    {
	      /* Not overflown anymore, remove */
	      *overflown_counter->jobject = NULL;
	      if (last_overflown_counter != NULL)
		{
		  last_overflown_counter->next = overflown_counter->next;
		}
	      else
		{
		  env->overflown_counter_list_head = overflown_counter->next;
		}
	      overflown_counter->next = env->overflown_counter_free_list;
	      env->overflown_counter_free_list = overflown_counter;

	      instance->lockword = _svmf_lockword_thinlock
		(env->thread.thinlock_id, recursive_count,
		 _svmf_lockword_get_extra_bits (lockword));

	      return;
	    }

	  if (last_overflown_counter != NULL)
	    {			/* Make it first */
	      last_overflown_counter->next = overflown_counter->next;
	      overflown_counter->next = env->overflown_counter_list_head;
	      env->overflown_counter_list_head = overflown_counter;
	    }

	  return;
	}
      overflown_counter = overflown_counter->next;
    }
  _svmm_fatal_error ("impossible control flow");
}

/*
----------------------------------------------------------------------
_svmm_stopping_java
----------------------------------------------------------------------
*/

#ifndef NDEBUG

#if defined(HAVE___func__)

#define _svmm_stopping_java(env)				\
_svmm_stopping_java_aux(env, __FILE__, __func__, __LINE__)

#elif defined(__FUNCTION__)

#define _svmm_stopping_java(env)				\
_svmm_stopping_java_aux(env, __FILE__, __FUNCTION__, __LINE__)

#else

#define _svmm_stopping_java(env)				\
_svmm_stopping_java_aux(env, __FILE__, "unknown", __LINE__)

#endif

#define _svmm_stopping_java_aux(env, file_, func_, line_)				\
do											\
{											\
  jint last_index = (env->stop_resume.last_index + 1) % SVM_STOP_RESUME_TRACE_SIZE;	\
  env->stop_resume.last_index = last_index;						\
  env->stop_resume.info[last_index].action = "stop";					\
  env->stop_resume.info[last_index].file = file_;					\
  env->stop_resume.info[last_index].func = func_;					\
  env->stop_resume.info[last_index].line = line_;					\
  env->stop_resume.info[last_index].entry = env->thread_status;				\
  env->stop_resume.info[last_index].exit = -1;						\
  _svmh_stopping_java(env);								\
  env->stop_resume.info[last_index].exit = env->thread_status;				\
} while(JNI_FALSE)

#else

#define _svmm_stopping_java(env)		\
_svmh_stopping_java(env)

#endif

svm_static void
_svmh_stopping_java (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;

#ifndef NDEBUG
  jint status = env->thread_status;

  if (status != SVM_THREAD_STATUS_RUNNING_JAVA &&
      status != SVM_THREAD_STATUS_HALT_REQUESTED)
    {
      jint last_index = env->stop_resume.last_index;
      jint current = last_index;

      do
	{
	  current = (current + 1) % SVM_STOP_RESUME_TRACE_SIZE;

	  if (env->stop_resume.info[current].action != NULL)
	    {
	      _svmf_printf (env, stderr, "%s %s %s %d %d %d\n",
			    env->stop_resume.info[current].action,
			    env->stop_resume.info[current].file,
			    env->stop_resume.info[current].func,
			    env->stop_resume.info[current].line,
			    env->stop_resume.info[current].entry,
			    env->stop_resume.info[current].exit);
	      fflush (NULL);
	    }
	}
      while (current != last_index);
    }

  assert (status == SVM_THREAD_STATUS_RUNNING_JAVA ||
	  status == SVM_THREAD_STATUS_HALT_REQUESTED);
#endif

  if (!_svmm_compare_and_swap
      (env->thread_status, SVM_THREAD_STATUS_RUNNING_JAVA,
       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED))
    {
      _svmm_mutex_lock (vm->global_mutex);

      _svmf_halt_if_requested (env);

      assert (env->thread_status == SVM_THREAD_STATUS_RUNNING_JAVA);

      env->thread_status =
	SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED;

      _svmm_mutex_unlock ();
    }
}

/*
----------------------------------------------------------------------
_svmf_stopping_java_noblock
----------------------------------------------------------------------
*/
/* This special function return JNI_OK if it was able to stop java
 * or JNI_ERR if resuming is disallowed, in which case the caller
 * should call the normal _svmf_stopping_java()
 */

#define _svmm_stopping_java_noblock(env)		\
_svmh_stopping_java_noblock(env)

inline svm_static jint
_svmh_stopping_java_noblock (_svmt_JNIEnv *env)
{
#ifndef NDEBUG
  jint status = env->thread_status;
  assert (status == SVM_THREAD_STATUS_RUNNING_JAVA ||
	  status == SVM_THREAD_STATUS_HALT_REQUESTED);
#endif

  if (!_svmm_compare_and_swap
      (env->thread_status, SVM_THREAD_STATUS_RUNNING_JAVA,
       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED))
    {
      return JNI_ERR;
    }
  else
    {
      return JNI_OK;
    }
}

/*
----------------------------------------------------------------------
_svmm_resuming_java
----------------------------------------------------------------------
*/

#ifndef NDEBUG

#if defined(HAVE___func__)

#define _svmm_resuming_java(env)				\
_svmm_resuming_java_aux(env, __FILE__, __func__, __LINE__)

#elif defined(__FUNCTION__)

#define _svmm_resuming_java(env)				\
_svmm_resuming_java_aux(env, __FILE__, __FUNCTION__, __LINE__)

#else

#define _svmm_resuming_java(env)				\
_svmm_resuming_java_aux(env, __FILE__, "unknown", __LINE__)

#endif

#define _svmm_resuming_java_aux(env, file_, func_, line_)				\
do											\
{											\
  jint last_index = (env->stop_resume.last_index + 1) % SVM_STOP_RESUME_TRACE_SIZE;	\
  env->stop_resume.last_index = last_index;						\
  env->stop_resume.info[last_index].action = "resume";					\
  env->stop_resume.info[last_index].file = file_;					\
  env->stop_resume.info[last_index].func = func_;					\
  env->stop_resume.info[last_index].line = line_;					\
  env->stop_resume.info[last_index].entry = env->thread_status;				\
  env->stop_resume.info[last_index].exit = -1;						\
  _svmh_resuming_java(env);								\
  env->stop_resume.info[last_index].exit = env->thread_status;				\
} while(JNI_FALSE)

#else

#define _svmm_resuming_java(env)		\
_svmh_resuming_java(env)

#endif

svm_static void
_svmh_resuming_java (_svmt_JNIEnv *env)
{
  _svmt_JavaVM *vm = env->vm;

#ifndef NDEBUG
  jint status = env->thread_status;

  if (status != SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED &&
      status != SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED)
    {
      jint last_index = env->stop_resume.last_index;
      jint current = last_index;

      do
	{
	  current = (current + 1) % SVM_STOP_RESUME_TRACE_SIZE;

	  if (env->stop_resume.info[current].action != NULL)
	    {
	      _svmf_printf (env, stderr, "%s %s %s %d %d %d\n",
			    env->stop_resume.info[current].action,
			    env->stop_resume.info[current].file,
			    env->stop_resume.info[current].func,
			    env->stop_resume.info[current].line,
			    env->stop_resume.info[current].entry,
			    env->stop_resume.info[current].exit);
	      fflush (NULL);
	    }
	}
      while (current != last_index);
    }

  assert (status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED ||
	  status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);
#endif

  if (!_svmm_compare_and_swap
      (env->thread_status,
       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED,
       SVM_THREAD_STATUS_RUNNING_JAVA))
    {
      _svmm_mutex_lock (vm->global_mutex);

      while (env->thread_status ==
	     SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED)
	{
	  _svmm_cond_wait (env->wakeup_cond, vm->global_mutex);
	}

      assert (env->thread_status ==
	      SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED);

      env->thread_status = SVM_THREAD_STATUS_RUNNING_JAVA;

      _svmm_mutex_unlock ();
    }
}

/*
----------------------------------------------------------------------
_svmf_resuming_java_noblock
----------------------------------------------------------------------
*/
/* This special function return JNI_OK if it was able to resume java
 * or JNI_ERR if resuming is disallowed, in which case the caller
 * should call the normal _svmf_resuming_java()
 */

#define _svmm_resuming_java_noblock(env)		\
_svmh_resuming_java_noblock(env)

svm_static inline jint
_svmh_resuming_java_noblock (_svmt_JNIEnv *env)
{
#ifndef NDEBUG
  jint status = env->thread_status;
  assert (status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED ||
	  status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);
#endif

  if (!_svmm_compare_and_swap
      (env->thread_status,
       SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED,
       SVM_THREAD_STATUS_RUNNING_JAVA))
    {
      return JNI_ERR;
    }
  else
    {
      return JNI_OK;
    }
}

/*
----------------------------------------------------------------------
_svmf_deflate_or_transfer
----------------------------------------------------------------------
*/

inline svm_static jint
_svmf_deflate_or_transfer (_svmt_JNIEnv *env, _svmt_JNIEnv *main_body)
{
  _svmt_JNIEnv *target_thread = NULL;
  _svmt_object_instance *instance = *main_body->body.jobject;
  _svmt_word old_lockword = instance->lockword;

  /* [ * we must be "running java" while calling this subroutine * ] */
#ifndef NDEBUG
  jint status = env->thread_status;
  assert (status == SVM_THREAD_STATUS_RUNNING_JAVA ||
	  status == SVM_THREAD_STATUS_HALT_REQUESTED);
#endif
  /* [ * main body mutex must be acquired around * ]
   * [   calls to this subroutine                * ] */
  assert (_svmm_mutex_trylock (main_body->body.mutex) == EBUSY);

  /* ! < main body has blocked threads > */
  if (_svmm_dll_empty (main_body, blocked))
    {
      /* ! < main body has waiting threads > */
      if (_svmm_dll_empty (main_body, wait))
	{
	  /* deflate (might require overflown rec. cnt. handling) */
	  /* Note that we do NOT decrement the counter here! */
	  jint recursive_count = main_body->body.recursive_count - 1;
	  /* We should be the encloser when deflating. */
	  assert (env->locking.encloses == main_body);

	  if (recursive_count >= SVM_THINLOCK_MAX_RECURSIVE_COUNT)
	    {
	      /* overflown counter in thinlock - create */
	      _svmt_overflown_counter *overflown_counter =
		env->overflown_counter_free_list;
	      /* Get us a counter first */
	      if (overflown_counter == NULL)
		{
		  if (_svmm_gzalloc_overflown_counter
		      (env, overflown_counter) != JNI_OK)
		    {
		      return JNI_ERR;	/* OOM */
		    }
		  if (_svmm_new_native_local
		      (env, overflown_counter->jobject) != JNI_OK)
		    {
		      _svmm_gzfree_overflown_counter (overflown_counter);
		      return JNI_ERR;	/* OOM */
		    }
		}
	      else
		{
		  /* We took it away from the free list */
		  env->overflown_counter_free_list = overflown_counter->next;
		}

	      *overflown_counter->jobject = *main_body->body.jobject;
	      overflown_counter->count = recursive_count;

	      overflown_counter->next = env->overflown_counter_list_head;
	      env->overflown_counter_list_head = overflown_counter;

	      recursive_count = SVM_THINLOCK_MAX_RECURSIVE_COUNT;
	    }
	  /* [ * set body "attached" flag to false        ] */
	  main_body->body.attached = JNI_FALSE;
	  /* [ * update head to be thin (detach the body) ] */
	  instance->lockword = _svmf_lockword_thinlock
	    (main_body->body.owner->thread.thinlock_id, recursive_count,
	     _svmf_lockword_get_extra_bits (old_lockword));

	  /* [ * clean and free the enclosed body ] */
	  *main_body->body.jobject = NULL;
	  main_body->body.owner = NULL;
	  _svmm_assert_free_body (main_body);

	  return JNI_OK;
	}
      else
	{
	  /* [ * pick the farthest waiting thread as Target ] */
	  target_thread = _svmm_dll_last (main_body, wait);
	  goto swap_thread_bodies;
	}
    }
  else
    {
      /* [ * pick the next blocked thread as Target ] */
      target_thread = _svmm_dll_first (main_body, blocked);
    swap_thread_bodies:
      assert (target_thread != NULL);
      /* ! < Target is main body encloser already > */
      if (target_thread->locking.encloses != main_body)
	{
	  /* [ * exchange encloseness of:              ]
	   * [   - the free body Target encloses with  ]
	   * [   - the main body;                      ]
	   * [   update enclosers locations in bodies  ] */
	  _svmm_assert_free_body (target_thread->locking.encloses);
	  _svmm_swap_thread_bodies (main_body->body.encloser, target_thread);
	}
    }

  _svmm_assert_not_encloser (instance);

  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_acquire_body
----------------------------------------------------------------------
*/

inline svm_static jint
_svmf_acquire_body (_svmt_JNIEnv *env, jint recursive_count,
		    _svmt_JNIEnv *wanted_body)
{
  _svmm_mutex_new_name (body_mutex);

#ifndef NDEBUG
  jint status = env->thread_status;
  /* [ * we must be "not running java" while calling this subroutine * ] */
  assert (status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED ||
	  status == SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);
#endif

  body_mutex = &wanted_body->body.mutex;

  assert (*wanted_body->body.jobject !=NULL);

  while (JNI_TRUE)
    {
      /* < wakeup reason is "moved" > */
      if (env->contention.wakeup_reason == SVM_WAKEUP_MOVED)
	{
	  return JNI_OK;
	}

      /* [ * "attached" of this body is set, body is attached ] */
      if (wanted_body->body.attached == JNI_TRUE)
	{
	  /* ! < can acquire main body ownership > */
	  if (wanted_body->body.owner == NULL)
	    {
	      break;
	    }
	}
      /* [ * wait (own thread CV, main body mutex)      ] */
      _svmm_cond_wait_v (env->locking.thread_cond, wanted_body->body.mutex);
    }

  /* [ * set main body owner to self             ] */
  wanted_body->body.owner = env;
  /* [ * restore recursive count                 ] */
  wanted_body->body.recursive_count = recursive_count;
  /* [ * remove self from main body blocked list ] */
  _svmm_dll_remove (wanted_body, blocked, env);
  /* Optimization: try to cheaply switch to running java. */
  if (__unlikely (_svmm_resuming_java_noblock (env) != JNI_OK))
    {
      body_mutex = &wanted_body->body.mutex;
      /* [ ? release main body mutex                 ] */
      _svmm_mutex_unlock_name (body_mutex);
      /* [ + transition to "running java"            ] */
      _svmm_resuming_java (env);
      /* [ ? acquire main body mutex                 ] */
      _svmm_mutex_lock_name (body_mutex);
    }

  /* { DEFLATE OR TRANSFER } */
  return _svmf_deflate_or_transfer (env, wanted_body);
}

/*
----------------------------------------------------------------------
_svmf_contention_handling
----------------------------------------------------------------------
*/

inline svm_static void
_svmf_contention_handling (_svmt_JNIEnv *env, _svmt_object_instance *instance)
{
  _svmt_JNIEnv *current_body;

  /* [ * acquire & release own thread mutex around calls to ]
   * [   this subroutine                                    ] */
  assert (_svmm_mutex_trylock (env->locking.thread_mutex) == EBUSY);

  /* ! < contention flag still set > */
  if (!env->contention.flag)
    {
      return;
    }

  /* [ * assert: semi-attached bodies list not empty) ] */
  assert (env->contention.semi_attached_list);

  /* [ * take the next unprocessed semi-attached body ]
   * [ * until all semi-attached bodies are processed ] */
  for (current_body = env->contention.semi_attached_list; current_body;
       current_body = current_body->contention.semi_attached_next)
    {
      /* < current body referst to the just released monitor > */
      if (*current_body->body.jobject == instance)
	{
	  THD3 ("CH-Floating %i(%i), instance: %p",
		current_body->body.encloser->thread.id,
		current_body->thread.id, instance);
	  /* [ * set body encloser/creator wakeup reason to "floating" ] */
	  current_body->body.encloser->contention.wakeup_reason =
	    SVM_WAKEUP_FLOATING;
	  _svmm_register_operation ('?', *current_body->body.jobject);
	  /* [ * signal the encloser/creator                           ] */
	  _svmm_cond_signal_v
	    (current_body, current_body->body.encloser->locking.thread_cond);
	}
      else
	{
	  /* inflate */
	  _svmt_word old_lockword = (*current_body->body.jobject)->lockword;
	  /* [ * assert (head is thin and we own it)                   ] */
	  THD3 ("CH-Inflate T%i(B%i), instance: %p",
		current_body->body.encloser->thread.id,
		current_body->thread.id, *current_body->body.jobject);
	  assert (_svmf_lockword_is_thin (old_lockword));
	  assert (_svmf_lockword_get_thinlock_id (old_lockword) ==
		  env->thread.thinlock_id);
	  _svmm_register_operation ('?', *current_body->body.jobject);
	  /* [ * set semi-attached body encloser wakeup reason to      ]
	   * [   "attached"                                            ] */
	  current_body->body.encloser->contention.wakeup_reason =
	    SVM_WAKEUP_ATTACHED;
	  /* [ * set self as the body owner                            ] */
	  current_body->body.owner = env;
	  /* [ * set recursive count to what thin head contained       ] */
	  current_body->body.recursive_count =
	    _svmf_lockword_get_thinlock_recursive_count (old_lockword) + 1;
	  /* [ * make head fat and pointing to the body                ] */
	  (*current_body->body.jobject)->lockword =
	    _svmf_lockword_fatlock
	    (current_body->thread.id,
	     _svmf_lockword_get_extra_bits (old_lockword));
	  /* [ * set body "attached" flag to true                      ] */
	  current_body->body.attached = JNI_TRUE;
	}
    }
  /* [ * empty the list of semi-attached bodies ] */
  env->contention.semi_attached_list = NULL;
  /* [ * zero own contention flag               ] */
  env->contention.flag = 0;

  /* NOTE: This assertion is clearly wrong when called from optimized
   * version of j.l.wait(), inflate_in_own_cell case */
#if 0
  _svmm_assert_not_encloser (instance);
#endif
}

/*
----------------------------------------------------------------------
_svmf_enter_object_monitor
----------------------------------------------------------------------
*/

inline svm_static jint
_svmf_enter_object_monitor (_svmt_JNIEnv *env,
			    _svmt_object_instance *instance)
{
  _svmt_JavaVM *vm = env->vm;
  _svmt_word old_lockword;

  assert (instance != NULL);

retry:
  old_lockword = instance->lockword;

  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED);
  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);

  {
    _svmt_word new_lockword = env->thread.thinlock_id |
      _svmf_lockword_get_extra_bits (old_lockword);

    /* [ CAS ]   < success > */
    if (__likely (_svmm_compare_and_swap
		  (instance->lockword,
		   _svmf_lockword_get_extra_bits (old_lockword),
		   new_lockword)))
      {
	/* thin lock acquired */
	return JNI_OK;
      }
  }

  /* Either the object is already locked, or the lock is inflated. */

  /* < head is thin > */
  if (_svmf_lockword_is_thin (old_lockword))
    {
      _svmm_mutex_new_name (owner_mutex);
      /* it is a thin lock */

      /* < ours > */
      if (__likely (_svmf_lockword_get_thinlock_id (old_lockword) ==
		    env->thread.thinlock_id))
	{
	  /* the thinlock is already owned by the current thread */
	  /* { RECURSIVE COUNT INCREMENT } */

	  jint recursive_count =
	    _svmf_lockword_get_thinlock_recursive_count (old_lockword);

	  assert (recursive_count <= SVM_THINLOCK_MAX_RECURSIVE_COUNT);

	  if (__likely (++recursive_count < SVM_THINLOCK_MAX_RECURSIVE_COUNT))
	    {
	      instance->lockword = _svmf_lockword_thinlock
		(env->thread.thinlock_id, recursive_count,
		 _svmf_lockword_get_extra_bits (old_lockword));

	      /* thin lock re-entered */
	      return JNI_OK;
	    }
	  else
	    {			/* Overflow handling */
	      return _svmf_lock_thin_increment_overflow
		(env, instance, old_lockword, recursive_count);
	    }
	}
      else
	{
	  /* Not < ours > */
	  /* contention: the thinlock is owned by another thread */

	  jint owner_id = _svmf_lockword_get_thread_id (old_lockword);
	  _svmt_JNIEnv *owner = vm->threads.array[owner_id];
	  _svmt_JNIEnv *semi_attached = NULL;
	  _svmt_word old_flag;
	  _svmt_word refetched_lockword;

	  /* If we read "old_lockword" before the thinlock was
	   * actually acquired by another thread, retry. */
	  if (__unlikely (owner == NULL))
	    {
	      goto retry;
	    }

	  owner_mutex = &owner->locking.thread_mutex;
	  /* notify owning thread that inflation is requested */
	  /* [ * acquire owner thread mutex ]
	   * [ * set contention bit         ] */
	  _svmm_mutex_lock_name (owner_mutex);
	  _svmm_set_instance (instance);

	  old_flag = owner->contention.flag;

	  owner->contention.flag = 1;

	  /* re-fetch the instance's lockword, now that the thread has
	   * been notified */
	  refetched_lockword = instance->lockword;

	  _svmm_count_inc ("ME-cont-ntflag");
	  /* ! < monitor owner changed > */
	  if (__likely (_svmf_lockword_is_thin (refetched_lockword)
			&& _svmf_lockword_get_thinlock_id (refetched_lockword)
			== owner->thread.thinlock_id))
/***	  if (__likely (refetched_lockword == old_lockword)) */
	    {
	      _svmm_assert_free_body (env->locking.encloses);
	    retry_semi_attached:
	      /* the thinlock owner is still the same and has been notified */
	      /* < has semi-attached body for the wanted monitor > */
	      for (semi_attached = owner->contention.semi_attached_list;
		   semi_attached;
		   semi_attached =
		   semi_attached->contention.semi_attached_next)
		{
		  if (*(semi_attached->body.jobject) == instance)
		    {
		      break;
		    }
		}
	      /* < has semi-attached body ... > _continued_ */
	      if (semi_attached)
		{
		  /* [ * set own wakeup reason to "normal" ] */
		  env->contention.wakeup_reason = SVM_WAKEUP_NORMAL;
		  /* [ * add self to the blocked list of the existing ]
		   * [   semi-attached body                           ] */
		  _svmm_dll_append (semi_attached, blocked, env);
		  /* < enclosed (floating) body has blocked threads > */
		  if (!_svmm_dll_empty (env->locking.encloses, blocked))
		    {
		      _svmt_JNIEnv *target_thread;
		      _svmt_JNIEnv *enclosed_body = env->locking.encloses;

		      /* [ * acquire enclosed body mutex           ] */
		      _svmm_mutex_lock_v (enclosed_body->body.mutex);

		      /* [ * append the list of threads blocked on ]
		       * [   the enclosed body to the main body    ] */

		      _svmm_dll_append_list (semi_attached, enclosed_body,
					     blocked);
		      /* Tricky: we've already appended the list of blocked
		       * threads to another list, yet we hold the pointer
		       * to the thread at the old head, and we'll traverse
		       * the list until the end safely. */
		      for (target_thread =
			   env->locking.encloses->body.list_blocked_head;
			   target_thread;
			   target_thread = target_thread->locking.next)
			{
			  /* [ * take the next blocked thread Tb  ] */
			  /* [ * set Tb action flag to "moved"    ] */
			  target_thread->contention.wakeup_reason =
			    SVM_WAKEUP_MOVED;
			  /* Note: this is the only place where one thread
			   * moves another to a different blocked list */
			}
		      /* [ * clean and free the enclosed body ] */
		      enclosed_body->body.list_blocked_head = NULL;
		      enclosed_body->body.list_blocked_tail = NULL;
		      *enclosed_body->body.jobject = NULL;
		      _svmm_assert_free_body (enclosed_body);
		      /* [ * free enclosed body mutex         ] */
		      _svmm_mutex_unlock_v ();
		    }

		  {
		    _svmm_mutex_new_name (main_body_mutex);
		    /* [ * release owner thread mutex          ] */
		    _svmm_mutex_unlock_name (owner_mutex);
		  retry_moved1:
		    /* [ * transition to "not running java"    ] */
		    _svmm_stopping_java (env);
		    /* [ * acquire main body mutex             ] */
		    main_body_mutex = &semi_attached->body.mutex;
		    _svmm_mutex_lock_name (main_body_mutex);
		    assert (*semi_attached->body.jobject != NULL);
		    /* { ACQUIRE BODY } */
		    _svmf_acquire_body (env, 1, semi_attached);
		    /* < ! wakeup reason is "moved" > */
		    if (env->contention.wakeup_reason == SVM_WAKEUP_MOVED)
		      {
			_svmt_word body_index;
			/* The move can happen at most once intentionally,
			 * BUT many times due to spurious wakeups! */
			/* [ * set own wakeup reason to "normal" ] */
			env->contention.wakeup_reason = SVM_WAKEUP_NORMAL;
			/* [ * release obsolete main body mutex  ] */
			_svmm_mutex_unlock_name (main_body_mutex);
			/* [ * transistion to "running java"     ] */
			_svmm_resuming_java (env);
			/* We need to set variables to up-to-date values
			 * before retrying.  That is why we need to 
			 * switch back to "running java" for a moment. */
			old_lockword = instance->lockword;
			assert (!_svmf_lockword_is_thin (old_lockword));
			owner_id = _svmf_lockword_get_thread_id
			  (old_lockword);
			body_index = _svmf_lockword_get_fatlock_index
			  (old_lockword);
			/* Unless wakeup in acquire_body was spurious
			 * this body is not "semi attached" anymore,
			 * but is "attached". */
			semi_attached = vm->threads.array[body_index];
			assert (*semi_attached->body.jobject != NULL);

			goto retry_moved1;
		      }
		    assert (env->contention.wakeup_reason ==
			    SVM_WAKEUP_NORMAL);
		    /* [ * release main body mutex             ] */
		    _svmm_mutex_unlock_name (main_body_mutex);
		    return JNI_OK;
		  }
		}
	    }
	  else
	    {
	      /* the lock is not thin anymore or the owner has
	       * changed, so restore the contention flag to its
	       * original value */
	      /* [ * revert contention bit ] */
	      owner->contention.flag = old_flag;
	      /* [ * release owner thread mutex ] */
	      _svmm_mutex_unlock_name (owner_mutex);
	      /* retry locking */
	      goto retry;
	    }

	  {
	    /* Owner notified, No semi-attached exists yet */
	    /* [ * use enclosed free body to create new semi-attached ]
	     * [   body.                                              ] */
	    /* NOTE: the body might not be free, if we jumped back from
	     * goto retry_semi_attached.  There might be blocked threads
	     * on our floating body. */
	    assert (env->locking.encloses->body.owner == NULL);
	    assert (env->locking.encloses->body.list_wait_head == NULL);
	    assert (env->locking.encloses->body.list_wait_tail == NULL);

	    /* [ * register new semi-attached body with monitor owner ] */
	    env->locking.encloses->contention.semi_attached_next =
	      owner->contention.semi_attached_list;
	    owner->contention.semi_attached_list = env->locking.encloses;
	    *env->locking.encloses->body.jobject = instance;
	    /* [ * add self to the blocked list              ] */
	    _svmm_dll_append (env->locking.encloses, blocked, env);
	    /* [ * set own wakeup reason to "spurious"           ] */
	    env->contention.wakeup_reason = SVM_WAKEUP_SPURIOUS;

	    /* Optimization: try to cheaply switch to not running java */
	    if (__unlikely (_svmm_stopping_java_noblock (env) != JNI_OK))
	      {
		/* [ ? release owner thread mutex ] */
		_svmm_mutex_unlock_name (owner_mutex);

		/* This one is tricky!  To avoid deadlocks, transition to
		   stopping java must be done while the owner's contention
		   mutex is released. */
		/* [ + transition to "not running java" ] */
		_svmm_stopping_java (env);

		/* [ ? recquire owner thread mutex ] */
		_svmm_mutex_lock_name (owner_mutex);
	      }

	    /* < wakeup reason is "spurious" > */
	    while (env->contention.wakeup_reason == SVM_WAKEUP_SPURIOUS)
	      {
		_svmm_cond_wait_v (env->locking.thread_cond,
				   owner->locking.thread_mutex);
	      }

	    /* [ * release "outdated" owner thread mutex ] */
	    _svmm_mutex_unlock_name (owner_mutex);

	    /* < wakeup reason is ... > */
	    if (env->contention.wakeup_reason == SVM_WAKEUP_ATTACHED)
	      {
		_svmt_JNIEnv *enclosed_body = env->locking.encloses;
		/* [ * set own wakeup reason to "normal" ] */
		env->contention.wakeup_reason = SVM_WAKEUP_NORMAL;
		/* [ * acquire main body mutex           ] */
		_svmm_mutex_lock_v (enclosed_body->body.mutex);
		/* { ACQUIRE BODY } */
		/* Nobody should change our encloseness while we're the
		 * first ones on the blocked list and encloser. */
		assert (enclosed_body == env->locking.encloses);
		_svmf_acquire_body (env, 1, enclosed_body);
		/* [ * release main body mutex           ] */
		_svmm_mutex_unlock_v ();
		/* ( DONE ) */
		return JNI_OK;
	      }

	    assert (env->contention.wakeup_reason == SVM_WAKEUP_FLOATING);
	    /* [ * set own wakeup reason to "normal" ] */
	    env->contention.wakeup_reason = SVM_WAKEUP_NORMAL;
	    /* [ * transition to "running java"      ] */
	    _svmm_resuming_java (env);
	    /* re-read instance as GC could have happened */
	    instance = *env->locking.encloses->body.jobject;

	    /* [ * remove self from blocked list     ] */
	    _svmm_dll_remove (env->locking.encloses, blocked, env);

	    /* < floating body blocked list empty > */
	    if (_svmm_dll_empty (env->locking.encloses, blocked))
	      {
		/* [ * clean and free enclosed body (destroy   ]
		 * [   floating body)                          ] */
		*env->locking.encloses->body.jobject = NULL;
		_svmm_assert_free_body (env->locking.encloses);
		goto retry;
	      }
	    {
	      _svmt_word new_lockword;
	    retry_floating:
	      /* [ * we enclose floating body with blocked threads ] */
	      assert (env->locking.encloses->body.list_blocked_head != NULL);
	      /* [ * set self as the main body owner               ] */
	      env->locking.encloses->body.owner = env;
	      /* re-read instance, GC could have happened */
	      instance = *env->locking.encloses->body.jobject;
	      old_lockword = instance->lockword;
	      /* prepare lockword to point to floating body */
	      new_lockword = _svmf_lockword_fatlock
		(env->locking.encloses->thread.id,
		 _svmf_lockword_get_extra_bits (old_lockword));

#ifndef NDEBUG
	      /* Head is either thin, or fat but not using our body */
	      if (!_svmf_lockword_is_thin (old_lockword))
		{
		  assert (_svmf_lockword_get_fatlock_index (old_lockword)
			  != env->locking.encloses->thread.id);
		}
#endif
	      /* [ * CAS                                           ] */
	      if (_svmm_compare_and_swap
		  (instance->lockword,
		   _svmf_lockword_get_extra_bits (old_lockword),
		   new_lockword))
		{		/* Yes (inflate) */
		  /* [ * pick next blocked thread as Tx            ] */
		  _svmt_JNIEnv *target_thread =
		    _svmm_dll_first (env->locking.encloses, blocked);
		  /* [ * acquire main body mutex               ] */
		  _svmm_mutex_lock_v (env->locking.encloses->body.mutex);
		  _svmm_assert_free_body (target_thread->locking.encloses);
		  assert (env->locking.encloses->body.list_wait_head == NULL);
		  assert (env->locking.encloses->body.list_wait_tail == NULL);
		  /* [ * set recursive count to 1                  ] */
		  env->locking.encloses->body.recursive_count = 1;
		  /* [ * set body "attached" flag to true          ] */
		  env->locking.encloses->body.attached = JNI_TRUE;
		  /* [ * exchange encloseness of:                  ]
		   * [   - the free body Tx, with                  ]
		   * [   - the floating body;                      ]
		   * [   and update encloser locations in bodies   ] */
		  _svmm_swap_thread_bodies (target_thread, env);
		  /* [ * release main body mutex                   ] */
		  _svmm_mutex_unlock_v ();

		  _svmm_assert_not_encloser (instance);
		  /* ( DONE ) */
		  return JNI_OK;
		}
	      else		/* CAS failed */
		{
		  /* [ * set the floating body owner to null       ] */
		  env->locking.encloses->body.owner = NULL;
		  /* < head is thin > */
		  if (_svmf_lockword_is_thin (old_lockword))
		    {
		      jboolean notified;
		      /* We need this extra check becase it might happen
		       * that the monitor is free and thin, but somebody
		       * modified some of other bits between when
		       * new_lockword was computed and CAS, and this
		       * was the reason CAS failed.
		       * (Think of the hash value bits) */
		      if (old_lockword != instance->lockword)
			{
			  goto retry_floating;
			}

		      owner_id = _svmf_lockword_get_thread_id (old_lockword);
		      owner = vm->threads.array[owner_id];

		      if (owner == NULL)
			{
			  goto retry_floating;
			}
		      /* [ * acquire thread mutex of monitor owner ]
		       * [ * set owner contention bit              ] */
		      _svmm_mutex_lock_v (owner->locking.thread_mutex);

		      old_flag = owner->contention.flag;

		      owner->contention.flag = 1;

		      /* re-fetch the instance's lockword, now that
		       * the thread has been notified */
		      refetched_lockword = instance->lockword;

		      /* ! < owner changed > */
		      if (_svmf_lockword_is_thin (refetched_lockword)
			  && _svmf_lockword_get_thinlock_id
			  (refetched_lockword) == owner->thread.thinlock_id)
			{
			  /* the thinlock owner is still the same
			   * and has been notified */
			  notified = JNI_TRUE;
			  /* This goto is a bit tricky because we're
			   * jumping back into the middle of something. */
			  owner_mutex = &owner->locking.thread_mutex;
			  goto retry_semi_attached;
			}
		      else
			{
			  notified = JNI_FALSE;
			  /* [ * revert contention bit ] */
			  owner->contention.flag = old_flag;
			}
		      /* [* release owner thread mutex ] */
		      _svmm_mutex_unlock_v ();

		      if (!notified)
			{
			  goto retry_floating;
			}
		    }
		  else		/* head is fat */
		    {
		      _svmm_mutex_new_name (wanted_body_mutex);
		      jboolean encloser_changed,
			fat_reacquire_needed = JNI_FALSE;
		      /* [ * acquire main body mutex ] */
		      _svmt_word wanted_body_index =
			_svmf_lockword_get_fatlock_index (old_lockword);
		      _svmt_JNIEnv *wanted_body =
			vm->threads.array[wanted_body_index];

		      wanted_body_mutex = &wanted_body->body.mutex;
		      _svmm_mutex_lock_name (wanted_body_mutex);

		      /* < main body still associated with this object > */
		      if (*wanted_body->body.jobject == instance)
			{
			  _svmt_JNIEnv *target_thread;
			  _svmt_JNIEnv *enclosed_body = env->locking.encloses;

			  assert (wanted_body != enclosed_body);
			  encloser_changed = JNI_FALSE;
			  /* [ * acquire enclosed body mutex      ] */
			  _svmm_mutex_lock_v (enclosed_body->body.mutex);

			  /* [ * append the list of threads blocked on ]
			   * [   the enclosed body to the main body    ] */
			  _svmm_dll_append_list (wanted_body, enclosed_body,
						 blocked);
			  /* Tricky: we've already appended the list of
			   * blocked threads to another list, yet we hold
			   * the pointer to the thread at the old head, and
			   * we'll traverse the list until the end safely. */
			  for (target_thread =
			       env->locking.encloses->body.list_blocked_head;
			       target_thread;
			       target_thread = target_thread->locking.next)
			    {
			      /* [ * take the next blocked thread Tb  ] */
			      /* [ * set Tb action flag to "moved"    ] */
			      target_thread->contention.wakeup_reason =
				SVM_WAKEUP_MOVED;
			    }
			  /* [ * clean and free the enclosed body ] */
			  enclosed_body->body.list_blocked_head = NULL;
			  enclosed_body->body.list_blocked_tail = NULL;
			  *enclosed_body->body.jobject = NULL;
			  _svmm_assert_free_body (enclosed_body);
			  /* [ * free enclosed body mutex         ] */
			  _svmm_mutex_unlock_v ();

			  /* < can acquire main body ownership > */
			  if (wanted_body->body.owner == NULL)
			    {
			      /* [ * set owner to self         ] */
			      wanted_body->body.owner = env;
			      /* [ * set recursive count to 1  ] */
			      wanted_body->body.recursive_count = 1;
			    }
			  else
			    {
			      /* [ * append self to the list of blocked ]
			       * [      threads                         ] */
			      _svmm_dll_append (wanted_body, blocked, env);
			      fat_reacquire_needed = JNI_TRUE;
			    }
			}
		      else
			{
			  encloser_changed = JNI_TRUE;
			}
		      /* [ * release main/wanted body mutex ] */
		      _svmm_mutex_unlock_name (wanted_body_mutex);

		      if (encloser_changed)
			{
			  goto retry_floating;
			}
		      
		      if (fat_reacquire_needed)
			{
			  jint result;
			  if (__unlikely (_svmm_stopping_java_noblock (env)
					  != JNI_OK))
			    {
			      /* [ ? release main thread mutex ] */
			      _svmm_mutex_unlock_name (wanted_body_mutex);
			      /* [ + transition to "not running java" ] */
			      _svmm_stopping_java (env);
			      /* [ ? acquire main thread mutex ] */
			      _svmm_mutex_lock_name (wanted_body_mutex);
			    }
			  /* { ACQUIRE BODY } */
			  result = _svmf_acquire_body (env, 1, wanted_body);
			  /* [ * release main body mutex          ] */
			  _svmm_mutex_unlock_name (wanted_body_mutex);
			  /* ( DONE ) */
			  return result;
			}
		    }
		}
	    }
	  }
	}
    }
  else
    {
      /* it is a fat lock */

      _svmt_word body_index = _svmf_lockword_get_fatlock_index (old_lockword);
      _svmt_JNIEnv *wanted_body = vm->threads.array[body_index];
      /* < main body is ours > */
      if (__likely (wanted_body->body.owner == env))
	{
	  /* [ * increment recusive count ] */
	  wanted_body->body.recursive_count++;
	  /* ( DONE ) */
	  return JNI_OK;
	}
      else
	{
	  jboolean body_changed, acquire_body_needed = JNI_FALSE;
	  _svmm_mutex_new_name (wanted_body_mutex);
	  wanted_body_mutex = &wanted_body->body.mutex;
	  /* [ * acquire body mutex ] */
	  _svmm_mutex_lock_name (wanted_body_mutex);

	  /* < body still associated with this object > */
	  if (__likely (*wanted_body->body.jobject == instance))
	    {
	      body_changed = JNI_FALSE;
	      /* < can acquire main body ownership > */
	      if (__likely (wanted_body->body.owner == NULL))
		{
		  /* [ * set main body owner to self ] */
		  wanted_body->body.owner = env;
		  /* [ * set recursive count to 1    ] */
		  wanted_body->body.recursive_count = 1;
		}
	      else
		{
		  /* [ * append self to the list of blocked threads ] */
		  _svmm_dll_append (wanted_body, blocked, env);
		  acquire_body_needed = JNI_TRUE;
		}
	    }
	  else
	    {
	      body_changed = JNI_TRUE;
	    }

	  if (__unlikely (acquire_body_needed))
	    {
	      /* Optimization: try to cheaply switch to not running java */
	      if (__unlikely (_svmm_stopping_java_noblock (env) != JNI_OK))
		{
		  /* [ ? release main body mutex          ] */
		  _svmm_mutex_unlock_name (wanted_body_mutex);
		  /* [ + transition to "not running java" ] */
		  _svmm_stopping_java (env);
		  /* [ ? acquire main body mutex          ] */
		  _svmm_mutex_lock_name (wanted_body_mutex);
		}
	      /* [ * FAT REACQUIRE * ] */
	      /* Note: we'll be running java when this call returns */
	      _svmf_acquire_body (env, 1, wanted_body);
	    }

	  /* [ * release main body mutex ] */
	  _svmm_mutex_unlock_name (wanted_body_mutex);

	  if (__unlikely (body_changed))
	    {
	      goto retry;
	    }
	}
    }

  _svmm_assert_not_encloser (instance);

  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_enter_object_monitor_non_blocking
----------------------------------------------------------------------
*/

inline svm_static jint
_svmh_enter_object_monitor_non_blocking (_svmt_JNIEnv *env,
					 _svmt_object_instance *instance,
					 jboolean *succeeded)
{
  _svmt_JavaVM *vm = env->vm;
  _svmt_word old_lockword;
  jboolean body_changed;

  assert (instance != NULL);

  _svmm_assert_not_encloser (instance);

retry:
  body_changed = JNI_FALSE;

  old_lockword = instance->lockword;

  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED);
  assert (env->thread_status !=
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_DISALLOWED);

  {
    _svmt_word new_lockword = env->thread.thinlock_id |
      _svmf_lockword_get_extra_bits (old_lockword);

    if (_svmm_compare_and_swap
	(instance->lockword, _svmf_lockword_get_extra_bits (old_lockword),
	 new_lockword))
      {
	/* thin lock acquired */
	*succeeded = JNI_TRUE;
	return JNI_OK;
      }
  }

  /* Either the object is already locked, or the lock is inflated. */

  if (_svmf_lockword_is_thin (old_lockword))
    {
      /* it is a thin lock */

      if (_svmf_lockword_get_thinlock_id (old_lockword) ==
	  env->thread.thinlock_id)
	{
	  /* the thinlock is already owned by the current thread */

	  jint recursive_count =
	    _svmf_lockword_get_thinlock_recursive_count (old_lockword);

	  assert (recursive_count <= SVM_THINLOCK_MAX_RECURSIVE_COUNT);

	  if (++recursive_count <= SVM_THINLOCK_MAX_RECURSIVE_COUNT)
	    {
	      instance->lockword = _svmf_lockword_thinlock
		(env->thread.thinlock_id, recursive_count,
		 _svmf_lockword_get_extra_bits (old_lockword));

	      /* thin lock re-entered */
	      *succeeded = JNI_TRUE;
	      return JNI_OK;
	    }
	  else
	    {			/* Overflow handling */
	      return _svmf_lock_thin_increment_overflow
		(env, instance, old_lockword, recursive_count);
	    }
	}
      else
	{
	  /* contention: the thinlock is owned by another thread */

	  *succeeded = JNI_FALSE;
	  return JNI_OK;
	}
    }
  else
    {
      /* it is a fat lock */

      _svmt_word body_index = _svmf_lockword_get_fatlock_index (old_lockword);
      _svmt_JNIEnv *main_body = vm->threads.array[body_index];
      jint status = JNI_OK;

      _svmm_mutex_lock_v (main_body->body.mutex);

      if (main_body->body.owner == env)
	{
	  main_body->body.recursive_count++;
	  *succeeded = JNI_TRUE;
	}
      else
	{
	  /* instance still associated with this fat lock? */
	  if (*main_body->body.jobject == instance)
	    {
	      if (main_body->body.owner == NULL)
		{
		  main_body->body.recursive_count = 1;
		  main_body->body.owner = env;
		  *succeeded = JNI_TRUE;
		}
	      else
		{
		  *succeeded = JNI_FALSE;
		}
	    }
	  else
	    {
	      body_changed = JNI_TRUE;
	    }
	}

      _svmm_mutex_unlock_v ();

      if (body_changed)
	{
	  goto retry;
	}

      if (status != JNI_OK)
	{
	  _svmf_error_OutOfMemoryError (env);
	  return JNI_ERR;
	}

      return JNI_OK;
    }
}

/*
----------------------------------------------------------------------
_svmf_exit_object_monitor
----------------------------------------------------------------------
*/

inline svm_static jint
_svmf_exit_object_monitor (_svmt_JNIEnv *env, _svmt_object_instance *instance)
{
  _svmt_JavaVM *vm = env->vm;
  _svmt_word old_lockword;

  jint recursive_count_on_entry;

  assert (instance != NULL);

  old_lockword = instance->lockword;

  /* < head is thin > */
  if (__likely (_svmf_lockword_is_thin (old_lockword)))
    {
      /* it is a thin lock */

      jint recursive_count;

      /* ! < thin head is ours > */
      if (_svmf_lockword_get_thinlock_id (old_lockword) !=
	  env->thread.thinlock_id)
	{
/*	  printf("ILLEGAL MSE 1\nTHREAD: %i/%i, FILE: %s:%i\n",
		 env->thread.id, __FILE__, __LINE__);
	  fflush(NULL); */
/*	  _svmf_dump_locking_structures(env); */
	  _svmf_error_IllegalMonitorStateException (env);
	  return JNI_ERR;
	}

      /* [ * decrement recursive count ] */
      recursive_count =
	_svmf_lockword_get_thinlock_recursive_count (old_lockword);
      recursive_count_on_entry = recursive_count + 1;

      /* < releasing > */
      if (__likely (recursive_count == 0))
	{			/* Thin lock release. */
	  /* [ * release thin head ] */
	  instance->lockword = _svmf_lockword_get_extra_bits (old_lockword);
	  goto handle_contention;
	}
      else if (__likely (recursive_count < SVM_THINLOCK_MAX_RECURSIVE_COUNT))
	{			/* Simple decrement. */
	  instance->lockword = _svmf_lockword_thinlock
	    (env->thread.thinlock_id, --recursive_count,
	     _svmf_lockword_get_extra_bits (old_lockword));
	  return JNI_OK;
	}
      else
	{			/* Overflown counter handling. */
	  _svmf_lock_thin_decrement_overflow (env, instance, old_lockword);
	  return JNI_OK;
	}
    }
  else
    {
      /* it is a fat lock */

      _svmt_word body_index = _svmf_lockword_get_fatlock_index (old_lockword);
      _svmt_JNIEnv *main_body = vm->threads.array[body_index];
      jint recursive_count;

      /* We must not be the encloser */
      _svmm_assert_not_encloser (instance);

      /* ! < main body ours > */
      if (__unlikely (main_body->body.owner != env))
	{
/* 	  printf("ILLEGAL MSE 2\nTHREAD: %i/%i, FILE: %s:%i\n",
		 env->thread.id, __FILE__, __LINE__);
	  fflush(NULL); */
/*	  _svmf_dump_locking_structures(env); */
	  _svmf_error_IllegalMonitorStateException (env);
	  return JNI_ERR;
	}
      recursive_count = --main_body->body.recursive_count;
      assert (recursive_count >= 0);

      /* ! < releasing > */
      if (__likely (recursive_count > 0))
	{
	  return JNI_OK;
	}

      /* [ * acquire main body mutex ] */
      _svmm_mutex_lock_v (main_body->body.mutex);

      /* < blocked threads list empty > */
      if (__unlikely (_svmm_dll_empty (main_body, blocked)))
	{
	  /* assert( # waiting > 0) */
	  assert (main_body->body.list_wait_head != NULL);
	  /* [ * set main body owner to null ] */
	  main_body->body.owner = NULL;
	}
      else
	{
	  _svmt_JNIEnv *target_thread;
	  /* [ * pick first blocked thread as Target ] */
	  target_thread = _svmm_dll_first (main_body, blocked);
	  /* < main body is enclosed by the Target thread > */
	  if (__likely (main_body == target_thread->locking.encloses))
	    {
	      /* [ * signal the Targe thread ] */
	      _svmm_cond_signal_v
		(main_body, target_thread->locking.thread_cond);
	      /* [ * give up fat lock's ownership ] */
	      main_body->body.owner = NULL;
	    }
	  else
	    {
	      /* [ * exchange encloseness of:                    ]
	       * [ - the free body Target encloses with          ]
	       * [ - with the main body;                         ]
	       * [   update enclosers locations in bodies        ] */
	      _svmm_assert_free_body (target_thread->locking.encloses);
	      _svmm_swap_thread_bodies
		(main_body->body.encloser, target_thread);
	      /* [ * set main body owner to null                 ] */
	      main_body->body.owner = NULL;
	      /* [ * signal thread Target                        ] */
	      _svmm_cond_signal_v
		(target_thread, target_thread->locking.thread_cond);
	    }
	}

      /* [ * release body mutex ] */
      _svmm_mutex_unlock_v ();

      _svmm_assert_not_encloser (instance);
      /* ( DONE ) */
      return JNI_OK;
    }

  assert (0);			/* we should never get here. */

handle_contention:

  /* < contention > */
  if (__unlikely (env->contention.flag))
    {
      _svmm_mutex_lock_v (env->locking.thread_mutex);

      _svmm_count_inc ("MX-cont-hdl");
      _svmf_contention_handling (env, instance);

      _svmm_mutex_unlock_v ();
    }

  /* we're done */
  _svmm_assert_not_encloser (instance);
  return JNI_OK;
}

/*
----------------------------------------------------------------------
_svmf_thread_start
----------------------------------------------------------------------
*/

svm_static void *
_svmf_thread_start (void *_env)
{
  _svmt_JNIEnv *env = (_svmt_JNIEnv *) _env;
  _svmt_JavaVM *vm = env->vm;

  assert (env->is_alive == JNI_TRUE);

  env->thread.pthread = pthread_self ();
  _svmf_set_current_env (env);

  _svmm_resuming_java (env);

  _svmm_invoke_static_virtualmachine_runthread (env);

  _svmm_mutex_lock (vm->global_mutex);

  _svmf_halt_if_requested (env);

  env->is_alive = JNI_FALSE;

  if (env->previous != NULL)
    {
      env->previous->next = env->next;
    }
  else
    {
      if (env->thread.is_daemon)
	{
	  vm->threads.system = env->next;
	}
      else
	{
	  vm->threads.user = env->next;
	}
    }

  if (env->next != NULL)
    {
      env->next->previous = env->previous;
    }

  _svmm_cond_signal (vm->threads.vm_destruction_cond);

  /* leak it for now... */

  _svmm_mutex_unlock ();

  return NULL;
}

/*
----------------------------------------------------------------------
_svmf_thread_native_start
----------------------------------------------------------------------
*/

svm_static _svmt_JNIEnv *
_svmf_thread_native_start (_svmt_JNIEnv *env, jobject threadInstance)
{
  _svmt_JNIEnv *result = NULL;
  _svmt_JNIEnv *new_env = NULL;
  _svmt_JavaVM *vm = env->vm;

  {
    jint status = JNI_OK;

    _svmm_mutex_lock (vm->global_mutex);

    _svmf_halt_if_requested (env);

    if (vm->threads.free_list != NULL)
      {
	new_env = vm->threads.free_list;
	assert (new_env->previous == NULL);

	vm->threads.free_list = new_env->next;
	if (vm->threads.free_list != NULL)
	  {
	    vm->threads.free_list->previous = NULL;
	  }

	new_env->next = vm->threads.user;
	if (new_env->next != NULL)
	  {
	    assert (new_env->next->previous == NULL);
	    new_env->next->previous = new_env;
	  }

	new_env->thread_status =
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED;
      }
    else if (vm->threads.next_thread_id <= SVM_MAX_THREAD_ID)
      {
	if (_svmm_gzalloc_env_no_exception (new_env) != JNI_OK)
	  {
	    goto unlock;
	  }

	new_env->interface = &_svmv_native_interface;
	new_env->vm = vm;

	new_env->next = vm->threads.user;
	vm->threads.user = new_env;
	if (new_env->next != NULL)
	  {
	    assert (new_env->next->previous == NULL);
	    new_env->next->previous = new_env;
	  }

	new_env->thread.id = vm->threads.next_thread_id++;
	_svmf_initialize_thinlock_id (new_env);
	vm->threads.array[new_env->thread.id] = new_env;

	new_env->locking.encloses = new_env;
	new_env->body.encloser = new_env;

	_svmm_cond_init (new_env->wakeup_cond);
	_svmm_cond_init (new_env->suspension.cond);
	_svmm_mutex_init (new_env->locking.thread_mutex);
	_svmm_cond_init (new_env->locking.thread_cond);
	_svmm_mutex_init (new_env->body.mutex);

	new_env->thread_status =
	  SVM_THREAD_STATUS_NOT_RUNNING_JAVA_RESUMING_ALLOWED;
      }
    else
      {
	status = JNI_ERR;
      }

  unlock:
    _svmm_mutex_unlock ();

    if (status != JNI_OK)
      {
	_svmf_error_OutOfMemoryError (env);
	goto end;
      }
  }

  if (_svmm_gzalloc_native_ref_no_exception (new_env->native_locals.list) !=
      JNI_OK)
    {
      /* we should be cleaning up!  todo ... */
      _svmf_error_OutOfMemoryError (env);
      goto end;
    }
  new_env->throwable = _svmf_cast_jobject_nref (new_env->native_locals.list);

  if (_svmm_gzalloc_native_ref_no_exception
      (new_env->native_locals.list->next) != JNI_OK)
    {
      /* we should be cleaning up!  todo ... */
      _svmf_error_OutOfMemoryError (env);
      goto end;
    }
  new_env->thread.thread_instance =
    _svmf_cast_jobject_nref (new_env->native_locals.list->next);
  *(new_env->thread.thread_instance) = *(threadInstance);

  if (_svmm_gzalloc_native_ref_no_exception
      (new_env->native_locals.list->next->next) != JNI_OK)
    {
      /* we should be cleaning up!  todo ... */
      _svmf_error_OutOfMemoryError (env);
      goto end;
    }
  new_env->locking.encloses->body.jobject =
    _svmf_cast_jobject_nref (new_env->native_locals.list->next->next);

  if (_svmm_gzalloc_native_ref_no_exception
      (new_env->native_locals.list->next->next->next) != JNI_OK)
    {
      /* we should be cleaning up!  todo ... */
      _svmf_error_OutOfMemoryError (env);
      goto end;
    }
  new_env->locking.waiting_on =
    _svmf_cast_jobject_nref (new_env->native_locals.list->next->next->next);

  if (_svmf_stack_init (new_env) != JNI_OK)
    {
      /* we should be cleaning up!  todo ... */
      *(env->throwable) = *(new_env->throwable);
      goto end;
    }

  /* start it! */
  new_env->is_alive = JNI_TRUE;
  if (pthread_create
      (&new_env->thread.pthread, NULL, &_svmf_thread_start, new_env) != 0)
    {
      /* something went wrong */
      /* we should be cleaning up!  todo ... */
      new_env->is_alive = JNI_FALSE;
      _svmf_error_InternalError (env);
      goto end;
    }

  /* it's started! */
  result = new_env;

end:
  return result;
}