#include "gpk_std.h"
#include <stdio.h>
#include <assert.h>
#include <math.h>
#include <stdlib.h>
#include <numerics.h>
#include <ctype.h>
#include "err.h"
#include <string.h>
#include "gpklib.h"
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <ieeefp.h>
# include <sunmath.h>
# include <ieeefp.h>
#include <ctype.h>
#include "popen.h"
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <memory.h>
#include <sys/wait.h>
#include <dlfcn.h>


#ifdef _REENTRANT
#include <thread.h>
#include <synch.h>
#endif

#define D	if(1)
#define DD	if(0)
#undef TEST

static int boomnum = 0, lastboomnum = 0;

void boom(int num)
{
 lastboomnum = boomnum;
 boomnum = num;
}


LOCAL void sigabrt(int sig)
{
 fprintf(stderr, "Abort, SEGV or BUS!!!! : %d\n", sig);
 fprintf(stderr, "boom= %d %d\n", boomnum, lastboomnum);
 fflush(stdout);
 fprintf(stderr, "boom= %d %d\n", boomnum, lastboomnum);
 exit(11);
}


C char sccs[] = "%W% %I%";
C char dir[] = SOURCEDIR;
static C int EXPSZ = 30;
static double timeout = 0;
C static unsigned Longtime = 2000;
static int nproc = 1;
static int iter_shift = 0;
static double T0 = quiet_nan(0);


LOCAL double max(double a, double b)	{return a>b ? a : b;}
LOCAL double xx(double x)	{return x*x;}



#ifdef _REENTRANT

class Mutex	{
	mutex_t m;

	public:
	Mutex()  { mutex_init(&m, USYNC_THREAD, 0); }
	~Mutex();
	void lock()	{boom(20000+thr_self()%100); mutex_lock(&m); }
	void unlock()	{mutex_unlock(&m); boom(20100); }
	int islocked();
	int trylock();
	};

int Mutex::islocked() {C int rv = mutex_trylock(&m); if(rv==0)	mutex_unlock(&m); return rv!=0;}
int Mutex::trylock() {C int rv = mutex_trylock(&m); return rv==0;}

Mutex::~Mutex()
{
 if(mutex_trylock(&m) != 0)	{
	fprintf(stderr, "ERR: Destroying locked mutex!");
	abort();
	}
 mutex_unlock(&m);
 mutex_destroy(&m);
}

#else

class Mutex	{
	public:
	Mutex()	{;}
	~Mutex()	{;}
	void lock()	{;}
	void unlock()	{;}
	};

#endif

LOCAL void copya(double *a, C double *b, int n)
{
 memcpy(a, b, n*sizeof(double));
}


LOCAL void p_i_error_check(int n, int tmp, unsigned l)
{
 boom(5000);
 if(n <= 0)
	gpk_err("Sorry. Can't set parameters until you specify N.\n", 0);
 if(tmp<0 || tmp>=n)	{
	char buf[100];
	sprintf(buf, "n=%d, index=%d, called from %u", n, tmp, l);
	gpk_err("parameter index out of bounds", buf);
	}
}





LOCAL void free_call(char **call)
{
 char **ca;
 for(ca=call; *ca; ca++)
	free(*ca);
 delete [] call;
}


LOCAL void warn_bad_exec(C char *msg, char **call, int copy, int stat)
{
 char buf[1024];
 int argc;
 boom(20101);
 for(argc=0; call[argc]; argc++)
	;
 boom(20103);

 fprintf(stderr, "INFO: opt.c:  for[copy=%d]   execv(%s %s %s %s %s %s)\n",
				copy,
				argc>0?call[0]:"", argc>1?call[1]:"",
				argc>2?call[2]:"", argc>3?call[3]:"",
				argc>5 ? "..." : "",
				argc>4 ? call[argc-1] : "");

 if(stat == (-1))
	sprintf(buf, "error=%s", strerror(errno));
 else if(WIFEXITED(stat))
	sprintf(buf, "exited with status=%d", WEXITSTATUS(stat));
 else if(WIFSIGNALED(stat))
	sprintf(buf, "terminated on signal %d", WTERMSIG(stat));
 else	sprintf(buf, "odd status: %d", stat);

 gpk_warn(msg, buf);
 free_call(call);
}


LOCAL void write_commandline(char **call, int errfd)
{
 int argc;
 for(argc=0;call[argc];argc++)	{
	char buf[1024];
	sprintf(buf, "'%s'%c", call[argc], call[argc+1] ? ' ' : '\n');
	write(errfd, buf, strlen(buf));
	}
}


struct hitlist_s	{
	time_t when;
	pid_t pid;
	int signal;
	struct hitlist_s *next;
	static Mutex m;
	};

Mutex hitlist_s::m;
static struct hitlist_s *hitlist = 0;


LOCAL void killer(int sig)
{
 struct hitlist_s *p;
 C int selflocked = hitlist->m.trylock();
 C time_t now = time(0);

 fprintf(stderr, "#WARN: Entered killer %d (boomnum=%d %d) at %ld [%s]\n",
			sig, lastboomnum, boomnum, now,
			selflocked?"Self-locked":"already-locked");

 for(p=hitlist; p; p=p->next)
	if(p->pid!=0 && p->when!=0)	{
		if(p->when<=now || sig!=(-1))	{
			int krv = kill(p->pid, p->signal);
			p->when = 0;

			fprintf(stderr, "#WARN: %s pid=%d\n",
						krv<0 ? "attempted kill of" : "killed",
						p->pid);
			sleep(3);
			kill(p->pid, SIGKILL);
			}
		}

 if(selflocked)
	hitlist->m.unlock();

 if(sig != (-1))
	exit(sig + 128);
}


LOCAL void add_to_hitlist(time_t when, pid_t victim, int sig)
{
 struct hitlist_s *p;
 assert(hitlist->m.islocked());
 for(p=hitlist; p; p=p->next)
	if((p->pid==victim && p->signal==sig) || p->pid==0)	{
		DD fprintf(stderr, "Recyclng victim=%d (was %ld ->%ld)\n",
						p->pid, p->when, when);
		break;
		}
 if(!p)	{
	p = new hitlist_s;
	DD fprintf(stderr, "New entry: v=%d when=%ld\n", victim, when);
	p->next = hitlist;
	hitlist = p;
	}

 if(p)	{
	p->when = when;
	p->pid = victim;
	p->signal = sig;
	}
}


static BOOL schedule_kill(pid_t victim, int interval)
{
 C time_t now = time(0);
 C time_t when = interval>0 ? now + interval : interval;
 hitlist->m.lock();

 struct hitlist_s *p;
 BOOL waskilled = FALSE;

 if(interval > 0)
	fprintf(stderr, "scheduling kill of %d for +%d seconds\n",
			victim, interval);
 else
	fprintf(stderr, "unscheduling kill of %d\n", victim);


 DD fprintf(stderr, "Hitlist:\n");
 for(p=hitlist; p; p=p->next)	{
	DD fprintf(stderr, "pid=%d\t\twhen=%ld\n", p->pid, p->when);
	if(p->pid==victim && p->when==0)	{
		DD fprintf(stderr, "WASKILLED: %d\n", victim);
		waskilled = TRUE;
		p->pid = 0;
		}
	}

 if(interval > 0)
	add_to_hitlist(when, victim, SIGTERM);
 else
	for(p=hitlist; p; p=p->next)
		if(p->pid == victim)	{
			DD fprintf(stderr, "freeing victim=%d\n", p->pid);
			p->when = 0;
			p->pid = 0;
			}


 int spid = 0;
 time_t soonest = now + Longtime;
 for(p=hitlist; p; p=p->next)
	if(p->pid!=0 && p->when!=0 && p->when<soonest)	{
		soonest = p->when;
		spid = p->pid;
		}

 if(spid!=0 && soonest>now)	{
	fprintf(stderr, "Next kill(%d) scheduled for +%ld seconds\n",
				spid, soonest-now);
	}
 else if(spid != 0)	{
	fprintf(stderr, "Next kill(%d) scheduled for now.\n", spid);
	killer(-1);
	}

 hitlist->m.unlock();

 boom(20102);

 return waskilled;
} 


class timestat_s	{
	double t;
	double tt;
	double n;

	public:
	double sigma() C;
	double avg() C;
	void add(double dt);
	timestat_s(): t(0), tt(0), n(0)	{;}

	C static double update_frac;
	};


static timestat_s *time_statistic = new timestat_s[1];


C double timestat_s::update_frac = 0.1;


double timestat_s::avg() C
{
 C double rv = n>0 ? t/n : infinity();
 return rv;
}


double timestat_s::sigma() C
{
 C double rv = n>2*update_frac ? sqrt((tt - xx(t)/n)/(n-update_frac)) : infinity();
 return rv;
}


void timestat_s::add(double dt)
{
 if(finite(dt))	{
	t += update_frac*(dt-t);
	tt += update_frac*(xx(dt) - tt);
	n += update_frac*(1-n);
	}
}


static int get_kill_interval(C timestat_s& ts)
{
 C double ivl = ts.sigma()*4 + 2*ts.avg();
 fprintf(stderr, "Kill interval=%g\n", ivl);
 if(finite(ivl))
	return 2 + (int)ivl;
 else return 0;
}



LOCAL char *append_int(char *a, int num)
{
 char buf[50];
 C char *bp;
 boom(5303);
 sprintf(buf, "%1d", num);
 for(bp=buf; *bp; bp++, a++)
	*a = *bp;
 return a;
}


LOCAL char *append_float(char *a, double num)
{
 char buf[50];
 C char *bp;
 boom(5303);
 sprintf(buf, "%g", num);
 for(bp=buf; *bp; bp++)
	if(!isspace(*bp))
		*a++ = *bp;
 return a;
}


static char **processor_label = new char* [nproc];




LOCAL int build_arglist(char **call, C char *b, int n,
				int copy, int iter)
{
 int argc = 0;
 C char *pop = 0;

 do	{
	char quote = 0;
	char argb[1000];
	char *a = argb;
	*a = '\0';
	while(*b && isspace(*b))
		b++;


	boom(5201);
	while(*b)	{
		assert(a-argb < sizeof(argb)-5);
		boom(5202);

		if((*b=='\\' && b[1]) || (*b=='%' && b[1]=='%'))	{
			boom(5300);
			*a++ = b[1];
			b++;
			}
		else if(!quote && (*b=='"' || *b=='\''))	{
			boom(5304);
			quote = *b;
			}
		else if(quote && *b==quote)	{
			boom(5305);
			quote = 0;
			}
		else if(*b=='%' && b[1]=='n')	{
			a = append_int(a, nproc);
			b++;
			}
		else if(*b=='%' && b[1]=='h')	{
			a = append_int(a, iter+iter_shift);
			b++;
			}
		else if(*b=='%' && b[1]=='d')	{
			a = append_int(a, copy);
			b++;
			}
		else if(*b=='%' && b[1]=='p')	{
			a = append_int(a, getpid());
			b++;
			}
		else if(*b=='%' && b[1]=='N')	{
			a = append_int(a, n);
			b++;
			}
		else if(*b=='%' && b[1]=='T')	{
			a = append_float(a, T0);
			b++;
			}
		else if(*b=='%' && b[1]=='s')	{
			assert(!pop);
			boom(5302);
			pop = b+2;
			if(!processor_label[copy])	{
				char buf[512];
				sprintf(buf, "No label for processor %d", copy);
				gpk_err(buf, 0);
				}
			b = processor_label[copy]-1;
			}
		else if(!quote && isspace(*b))	{
			boom(5306);
			break;
			}
		else	{
			boom(5307);
			*a++ = *b;
			}

		boom(5203);
		b++;
		boom(5211);
		if(!*b && pop)	{
			boom(5212);
			b = pop;
			pop = 0;
			boom(5204);
			}
		boom(5205);
		}
	boom(5206);
	*a = '\0';
	DD fprintf(stderr, "funk: arg[%d]: %s\n", argc, argb);
	if(*argb)	{
		call[argc++] = strdup(argb);
		assert(call[argc-1]);
		}
	boom(5207);

	} while(*b);
 call[argc] = 0;

 return argc;
}





LOCAL double do_exec_get(FILE *fp)
{
 BOOL found = FALSE;
 double rv = quiet_nan(0);
 char line[10000];
 while(fgets(line, sizeof(line), fp))	{
	double td1;
	boom(5505);

	if(chomp_newline(line) < 0)	{
		gpk_warn("Result missing terminating newline or too long:", line);
		break;
		}

	if(is_error_line(line))	{
		gpk_warn(line, 0);
		return quiet_nan(0);
		}

	trim_comments(line);

	if(is_blank_line(line))
		continue;

	if(!found && sscanf(line, "%lg", &td1) == 1)	{
		rv = td1;
		found++;
		}
	else if(!found)	{
		errno = 0;
		gpk_warn("can't parse output=", line);
		break;
		}
	}
 return rv;
}





static int nfnd = 0;

LOCAL double funk_exec(C double *p, int n, void *c, int nc, int copy, int iter)
{
 C char *cp = (C char *)c;
 boom(10002);
 char **call = new char* [strlen(cp)/2+2 + n + EXPSZ];
 boom(10003);
 assert(call);
 assert(c);
 assert(copy>=0 && copy<nproc);

 boom(5200);
 int argc = build_arglist(call, cp, n, copy, iter);

 boom(5208);

 int i;
 for(i=0; i<n; i++)	{
	char argb[1000];
	boom(6000+i);
	sprintf(argb, "%.12g", p[i]);
	boom(6100+i);
	call[argc++] = strdup(argb);
	assert(call[argc-1]);
	boom(6200+i);
	}
 call[argc] = 0;

 boom(5500);
 char errfile[256], errf_OK[256], errf_bad[256], errf_inf[256];
 sprintf(errfile, "stderr%1d", copy);
 sprintf(errf_OK, "stderr%1d.OK", copy);
 sprintf(errf_bad, "stderr%1d.Bad", copy);
 sprintf(errf_inf, "stderr%1d.Inf", copy);
 int errfd = open(errfile, O_WRONLY|O_CREAT|O_TRUNC, 0666);
 boom(5501);
 /* write_commandline(call, errfd); */

 time_t popen_time = time(0);
 boom(5502);
 struct popen_s fp = thr_popen(call, "r", errfd);
 close(errfd);
 schedule_kill(fp.pid, get_kill_interval(time_statistic[copy]));
 if(!fp.fp || !fp.pid)	{
	warn_bad_exec("popen failed", call, copy, 0);
	return quiet_nan(0);
	}

 boom(5503);
#ifdef _REENTRANT
 thr_yield();
#endif
 boom(5504);
 C double rv = do_exec_get(fp.fp);
 if(isnan(rv))
	kill(fp.pid, SIGTERM);
 boom(5600);
 {
 int pclrv;
 if((pclrv=thr_pclose(fp)) != 0)	{
	C BOOL waskilled = schedule_kill(fp.pid, 0);
	if(waskilled)
		time_statistic[copy].add( max(timeout,
						get_kill_interval(time_statistic[copy]) * 1.5));
	unlink(errf_bad);
	link(errfile, errf_bad);
	unlink(errfile);
	warn_bad_exec(waskilled ? "process killed-pclose" : "error on pclose",
			call, copy, pclrv);
	return quiet_nan(0);
	}
 }


 if(schedule_kill(fp.pid, 0))	{
	warn_bad_exec("process killed", call, copy, 0);
	time_statistic[copy].add( max(timeout,
					get_kill_interval(time_statistic[copy]) * 1.5));
	unlink(errf_bad);
	link(errfile, errf_bad);
	unlink(errfile);
	return quiet_nan(0);
	}

 if(isnan(rv))	{
	unlink(errf_bad);
	link(errfile, errf_bad);
	unlink(errfile);

	C char *s = "NaN or no value returned from call:";

	if(nfnd++ > 2*nproc)
		gpk_err(s, call[0]);
	else	{
		warn_bad_exec(s, call, copy, 0);
		return quiet_nan(0);
		}
	/* NOTREACHED */
	}
 else	{
	nfnd = 0;
	}

 if(finite(rv))	{	/* Inf may well take much less time... */
	time_statistic[copy].add(max(timeout, difftime(time(0), popen_time)) );
	unlink(errf_OK);
	link(errfile, errf_OK);
	unlink(errfile);
	}
 else	{
	unlink(errf_inf);
	link(errfile, errf_inf);
	unlink(errfile);
	}

 free_call(call);

 fprintf(stderr, "\t[copy=%d]===-> %g\n", copy, rv);

 return rv;
}


typedef double (*chi_squared_t)(char *C*, C double *, int);
LOCAL void *dll_handle = 0;
LOCAL char *dll_name = 0, *dll_init = 0;
LOCAL chi_squared_t dll_pointer = 0;

extern _etext;

LOCAL double funk_dll(C double *p, int n, void *c, int nc, int copy, int iter)
{
 assert(c);
 C char *cp = (C char *)c;
 boom(5209);
 char **call = new char* [strlen(cp)/2+2 + EXPSZ];
 boom(5210);
 assert(call);
 assert(copy>=0 && copy<nproc);

 int argc = build_arglist(call, cp, n, copy, iter);
 boom(5213);
 time_t popen_time = time(0);
 assert(dll_pointer);
 boom(5214);
 C double rv = (*dll_pointer)(call, p, n);
 boom(5215);

 time_statistic[copy].add( max(timeout,
				get_kill_interval(time_statistic[copy]) * 1.5));
 boom(5216);
 free_call(call);
 boom(5217);

 fprintf(stderr, "\t[copy=%d]===-> %g\n", copy, rv);
 boom(5218);

 return rv;
}

C char *help[] = {
"Usage: opt control_file >results",
"standard error has lots of debugging stuff. No argument, or '-' as",
"the control file causes it to be read from the standard input.",
"",
"OPT: generic optimizer(minimizer) of multiparameter functions.",
"",
"Opt uses a variant on a direction set method to optimize anything.",
"It executes a subprocess, reads a result off the standard output of that",
"subprocess, and tweaks the parameters (passed as arguments to the",
"subprocess) to make the output decrease.",
"It produces the file 'amoeba.tmp', containing a history of the current best set",
"of parameters (parameters are blank separated on a line, each line corresponds",
"to one iteration).  It also produces 'amoeba.dirset', which contains a history",
"of the best estimate for the direction set.",
"The standard output is a list of control statements (defined below) suitable",
"for feeding back into the standard input, to restart the optimization.",
"",
"It takes a list of control statements and intitial parameters on the standard",
"input.  These are formatted as follows:",
"Blank lines are ignored.",
"Comments are ignored (comments run from the first pound sign that begins a word",
"    to the end of the line).",
"A line beginning with 'ERR:' will stop the program and cause it to copy",
"    that error message to its output.",
"Then it consists of lines in the form ' A = b' (white space optional).",
"N=### (number of parameters to be optimized).",
"TOL=###.#  (controls when program stops).  Termination occurs when the average",
"    improvement per step is less than TOL/N^2.  For a well-behaved function,",
"    that means you are probably within TOL of the real minimum.",
"ITMAX=###  (maximum number of iterations).  An alternative means of stopping",
"   the optimization.  FYI: the function is called about 10 times per iteration.",
"ITMAX=###*N (see above).",
"ITSHIFT=#### (shift iteration for %h).  The ITSHIFT value is added to the iteration count,",
"   and substituted for %h in FUNC.  This is useful when re-starting an optimization,",
"   especially if chi-squared has some time dependence.",
"FUNC= <anything>",
"   Everything after the equal sign will be passed to execv() for execution,",
"   followed by N numeric parameters.  Note that execv() does not use a search",
"   path - you must specify an exact path name.  This program must print a number on",
"   its standard output: this number is what opt minimizes.",
"   Blank lines are ignored, as are comments (from # at word beginning to end of line).",
"   Only the first number is read, anything else is ignored",
"   If the first non-blank, non-comment is not a number,",
"   opt terminates in an error (though 'Inf' is treated properly as infinity).",
"   If the program terminates with a non-zero error code, any output will be",
"   ignored, and the program re-run (probably with different parameters).",
"   Finally, a line beginning with 'ERR:' will stop opt, and be passed out on the",
"   standard output.  See below for multiprocessors.  Several substitutions are",
"	defined: %n %N %s %d expand, and %% becomes %.  See elsewhere.", 
"   If the FUNC string contains:",
"   ... '%n' the number of processors is substituted.",
"   ... '%N' the number of parameters is substituted.",
"   ... '%h' the number of successful iterations is substituted, where",
"      successful is defined a la Monte-Carlo.",
"   ... '%p' the process-id number of the optimizer is substituted.",
"   ... '%d' the processor index [0,%n-1] is substituted.",
"   ... '%s' the processor's label is substituted.",
"   ... '%%' results in a percent sign.",
"  Quoting is done with single or double quotes, or with backslashes.  Only",
"    backslashes prevent the above '%' expansions.",
"p#=##.#+-##.# (setting a parameter).",
"   There are N parameters; this sets the initial value and a reasonable guess for",
"   the range over which the algorithm should start its search.  Parameters range",
"   from p0, p1, ... pN-1.  There must be no space before the '+-'.",
"   If DIRSET is specified, it will override the ranges.",
"FIRSTPRM=### just adjusts where the optimization begins work.",
"FIRSTPRM=RANDOM (no spaces allowed) begins with a randomly chosen parameter.",
"FIRSTPRM=SHUFFLE (no spaces allowed) shuffles the order of the direction set",
"   before beginning to run.  Useful for restarting an optimization.",
"ORDERIZE=filename  adjusts the order of the optimization.",
"   The file is read for lines containing [pP]####, and the order of fitting is",
"   arranged so that those parameters mentioned nearby in the file are worked on",
"   together.",
"DIRSET=filename  uses the direction set stored in the file (previously copied",
"   from amoeba.dirset), to allow the fitting program to pick up where it left off.",
"   The file format is mildly obscure.",
"INTEREST=filename lets you set which parameters are intrinsically more interesting.",
"   This is a two column file, where the first column is a parameter number",
"   (optionally preceeded by [pP] and the second column is a float.",
"   useful values are in the",
"   range -10 to 10, and default to zero if a parameter is not mentioned.",
"   Set the default with a  is '*', 'p*', or 'P*' in the first column.",
"   The program will eventually generate its own estimates of what's interesting,",
"   but, initially, the bigger the interest number, the more often that parameter",
"   will be changed.",
"NPROC=###  sets the number of processors that will run in parallel.",
"LABEL###= <anything>  stores away a string to label or name the numbered processor.",
"   If the FUNC string contains '%s' the value of the appropriate label is substituted.",
"   If the FUNC string contains '%d' the processor number is substituted.",
"TIMEOUT=###  sets the minimum timeout period before FUNC is killed.",
"   If FUNC is killed, the optimization will continue, re-running FUNC.",
"   Enough timeouts or errors in a row will eventually cause termination.",
"   By default, the timeout interval is set based on history.",
"T0=###.###  Sets the 'temperature' of the simulated annealing process.",
"   This allows the optimization to escape from local minima by allowing",
"   small upwards steps to be taken.   Upwards steps of size around T0 are",
"   accepted (with exponentially decreasing probability), while downward steps",
"   are always accepted.   Note that with N parameters, when you are simulated",
"   annealing, you will end up something like N*T0 above the true minimum.",
"",
"NOTES:",
"The way to run on different machines is to set",
"FUNC= rsh -n %s ..., and then set LABEL* to the names of the machines.",
0
   };



LOCAL void print_help(void)
{
 C char **h;
 for(h=help; *h; h++)
	fprintf(stderr, "%s\n", *h);
 fprintf(stderr, "version= %s, source code in %s/%s\n", sccs, dir, __FILE__);
}



enum linepos_e	{START, SIZELINE, DIRSETLINE};



LOCAL void read_dirset(double **p, int n, C char *file, double dely)
{
 double dy, scalefac = 1;
 double *sz = new double[n];
 assert(sz);
 char line[10000];
 FILE *fp = fopen(file, "r");
 if(!fp)
	gpk_err("Can't open dirset file ", file);

 BOOL gotsizes = FALSE;
 int k = 0;
 enum linepos_e pos = START;
 while(fgets(line, sizeof(line), fp))	{
	if(strncmp(line, "# sizes ", strlen("# sizes ")) == 0)	{
		pos = SIZELINE;
		gotsizes = FALSE;
		}
	else if(sscanf(line, "Dely = %lf", &dy) == 1)	{
		assert(dy > 0   && finite(dy));
		scalefac = dy/dely;
		}
	else if(strncmp(line, "# Direction set. ",
				strlen("# Direction set. ")) == 0)	{
		pos = DIRSETLINE;
		k = 0;
		}
	else if(line[0] == '#')
		;
	else if(is_blank_line(line))
		;
	else if(pos == SIZELINE)	{
		char **q = csplit(line, " \t");
		int m = csplitcount(q);
		int i;
		for(i=0;i<m && i<n;i++)	{
			sz[i] = atof(q[i]) * scalefac;
			assert(sz[i]>0 && finite(sz[i]));
			}
		pos = START;
		k = 0;
		gotsizes = TRUE;
		}
	else if(pos == DIRSETLINE)	{
		if(k >= n)
			gpk_err("Too many direction vectors in dirset file:", file);

		assert(gotsizes);
		char **q = csplit(line, ":");
		if(csplitcount(q) != 2)
			gpk_err("bad split on dirsetline", q[0]);
		C double l = atof(q[0]);
		assert(l>0 && finite(l));

		char **r = csplit(q[1], " \t");
		int m = csplitcount(r);
		int i;
		for(i=0;i<n && i<m;i++)	{
			C double f = atof(r[i]);
			assert(f>=(-1.1) && f<=1.1);
			p[k][i] = l*sqrt(sz[i])*f;
			}
		k++;
		}
	}

 {
  int i;
  for(i=0;i<n;i++)	{
	int j;
	p[n][i] = 0.0;
	for(j=0;j<n;j++)
		p[n][i] -= p[j][i];
	}
 }

 delete[] sz;
}


LOCAL void do_shuffle(double **p, int n)
{
	int pass;
	srand(time(0));
	D fprintf(stderr, "shuffle\n");
	for(pass=0;pass<n;pass++)	{
		C unsigned a = rand()%(n+1);
		C unsigned b = rand()%(n+1);
		assert(a>=0 && a<=n);
		assert(b>=0 && b<=n);
		int i;
		for(i=0;i<n;i++)	{
			C double tmp = p[a][i];
			p[a][i] = p[b][i];
			p[b][i] = tmp;
			}
		}
}


/* lines in the file contain numbers of the parameter to work on next;
it goes and finds which column in p has most effect on that parameter. */
LOCAL void do_orderize(double **p, C double *pc, int n, C char *file,
				int firstprm)
{
 int firstcol = 0;
 FILE *fp = fopen(file, "r");
 char line[1000];
 if(!fp)
	gpk_err("Can't open orderize file:", file);
 int *donep = new int[n];
 int *donespot = new int[n+1];
 double **tmp = double2alloc(n+1, n);
 int i;
 for(i=0; i<n; i++)
	donep[i] = FALSE;
 for(i=0; i<=n; i++)
	donespot[i] = FALSE;

 for(i=0; i<n && fgets(line, sizeof(line), fp);)	{
	trim_comments(line);
	char *l = line;
	char *s;
	while((s=strtok(l, "+-*/()=,;[]@%$&:'\"<>?{} \t\n\r"))!=0 && i<n)	{
		int x;
		if((sscanf(s, "p%d", &x)==1 || sscanf(s, "P%d", &x)==1)
				&& x>=0 && x<n && !donep[x])	{
			fprintf(stderr, "ORDERIZE: looking for p%d (n=%d, i=%d)...\n", x, n, i);
			int which, best = -1;
			double bv = 0;
			for(which=0; which<=n; which++)	{
				double tv;
				if(!donespot[which] && (tv=fabs(p[which][x]-pc[x])) > bv)	{
					bv = tv;
					best = which;
					}
				}
			donep[x] = TRUE;
			if(best >= 0)	{
				if(x == firstprm)
					firstcol = best;
				fprintf(stderr, "ORDERIZE: found col%d (n=%d, i=%d)...\n", best, n, i);
				assert(best >= 0);

				copya(tmp[i], p[best], n);
				donespot[best] = TRUE;
				i++;
				}
			}
		l = 0;
		}
	}
 fclose(fp);
 delete [] donep;

	/* copy over anything not yet mentioned... */
 int w = 0;
 for(; i<=n; i++)	{
	for(;w<=n && donespot[w]; w++)
		;
	fprintf(stderr, "ORDERIZE: cleanup col%d (n=%d, i=%d)...\n", w, n, i);
	assert(w <= n);
	copya(tmp[i], p[w], n);
	donespot[w] = TRUE;
	}

 for(i=0; i<=n; i++)
	copya(p[i], tmp[(i+firstcol)%(n+1)], n);

 double2free(tmp);
 delete [] donespot;
}




LOCAL float *do_interest(const char *ifile, int n)
{
 float *rv = new float [n];
 float x, def = 0;
 assert(rv);
 int i;
 for(i=0; i<n; i++)
	rv[i] = quiet_nanf(0);
 FILE *fp = fopen(ifile, "r");
 if(!fp)
	gpk_err("Can't open interest file: ", ifile);
 char line[1000];
 while(fgets(line, sizeof(line), fp))	{
	if((sscanf(line, "p%d %f", &i, &x)==2
			|| sscanf(line, "%d %f", &i, &x)==2)
			&& i>=0 && i<n)	{
		fprintf(stderr, "interest[%d] = %g\n", i, x);
		if(isnan(x))
			fprintf(stderr, "ERR: opt.c: do_interest: interest must not be NaN for p%d\n", i);

		rv[i] = x;
		}
	else if(sscanf(line, "p* %f", &x)==1
			|| sscanf(line, "* %f", &x)==1)	{
		fprintf(stderr, "interest[default] = %g\n", x);
		if(isnan(x))
			gpk_err("do_interest: default interest must not be NaN\n", 0);
		def = x;
		}
	}
 fclose(fp);

 for(i=0; i<n; i++)
	if(isnanf(rv[i]))
		rv[i] = def;

 return rv;
}



main(int argc, char **argv)
{
 BOOL shuffle = FALSE;
 int n = 0;
 char line[2000], *orderize = 0, *dirset = 0, *intrfile = 0;
 double **p = 0, *pc = 0;
 float *interest = 0;
 double ftol = quiet_nan(0);
 int itmax = 1000, firstprm = 0;
 char *fcn = 0;
 processor_label[0] = strdup("");
 FILE *inp = stdin;

 signal(SIGABRT, sigabrt);
 signal(SIGBUS, sigabrt);
 signal(SIGSEGV, sigabrt);

 boom(10001);

 if(argc>2 || (argc==2 && strcmp(argv[1], "^")==0))	{
	print_help();
	exit(1);
	}

 if(argc==2 && strcmp(argv[1], "-")!=0)	{
	inp = fopen(argv[1], "r");
	if(!inp)	{
		print_help();
		gpk_err("Can't open input file: ", argv[1]);
		}
	}

 while(fgets(line, sizeof(line), inp))	{
	int tmp, tmpn;
	char tmps[2000], tmps2[1000];
	double td1, td2;

	if(chomp_newline(line) < 0)
		gpk_err("Line too long - no newline", 0);

	trim_comments(line);

	if(is_blank_line(line))
		continue;

	D fprintf(stderr, "#L: %s\n", line);
	if(is_error_line(line))
		gpk_err(line, 0);

	if(sscanf(line, "N = %d", &tmp) == 1)	{
		if(p)
			gpk_err("N is already set", 0);
		if(tmp<2 || tmp>10000)
			gpk_err("Silly value of N", 0);

		n = tmp;
		D fprintf(stderr, "#alloc %d %d\n", n+1, n);
		p = double2alloc(n+1, n);
		pc = double1alloc(n);
		assert(double2test(p) == 0);
		assert(double1test(pc) == 0);
		double2set(p, 0.0);
		double1set(pc, quiet_nan(0));
		}
	else if(sscanf(line, "NPROC = %d", &tmp) == 1)	{
		if(tmp<1 || tmp>10000)
			gpk_err("Silly value of NPROC", 0);

		char **tl = new char * [tmp];
		assert(tl);
		int k;
		for(k=0; k<nproc && k<tmp; k++)
			tl[k] = processor_label[k];
		for(; k<nproc; k++)
			if(processor_label[k] && processor_label[k][0])	{
				char buf[128];
				sprintf(buf, "Wiped label%1d, as NPROC set to %d: ", k, tmp);
				gpk_warn(buf, processor_label[k]);
				free(processor_label[k]);
				}
		for(; k<tmp; k++)
			tl[k] = strdup("");
		nproc = tmp;
		delete [] processor_label;
		processor_label = tl;
		delete [] time_statistic;
		time_statistic = new timestat_s[nproc];
		assert(time_statistic);
		}
	else if((tmpn=sscanf(line, "LABEL%d = %[^\n]", &tmp, tmps)) >= 1)	{
		assert(tmp >= 0);
		assert(processor_label);
		if(tmp>=0 && tmp<nproc)	{
			if(processor_label[tmp])	{
				if(processor_label[tmp][0])
					gpk_warn("relabeling processor: ", line);
				free(processor_label[tmp]);
				}
			processor_label[tmp] = strdup(tmpn==2 ? tmps : "");
			}
		else
			gpk_warn("labeling a nonexistant processor.  Ignored:  ", line);
		}
	else if(sscanf(line, "TIMEOUT = %lf", &td1) == 1)
		timeout = td1;
	else if(sscanf(line, "TOL = %lf", &td1) == 1)
		ftol = td1;
	else if(sscanf(line, "ITMAX = %d %[*] %[N]", &tmp, tmps, tmps) == 3)
		itmax = tmp*n;
	else if(sscanf(line, "ITMAX = %d", &tmp) == 1)
		itmax = tmp;
	else if(sscanf(line, "ITSHIFT = %d", &tmp) == 1)
		iter_shift = tmp;
	else if(sscanf(line, "FIRSTPRM = %s", tmps) == 1)	{
		if(n <= 0)
			gpk_err("Sorry, FIRSTPRM cannot be set until N is set.", 0);
		if(strcmp(tmps, "RANDOM") == 0)
			firstprm = (5*(getpid()%n)+ 17*(getppid()%n) + time(0)%n)%n;
		else if( strcmp(tmps, "SHUFFLE") == 0)
			shuffle = TRUE;
		else if(sscanf(tmps, "%d", &tmp) == 1)
			firstprm = (tmp+n) % n;
		else	gpk_err("Bad FIRSTPRM= line:", line);
		}
	else if(sscanf(line, "T0 = %lf", &td1)==1)
		T0 = td1;
	else if(sscanf(line, "DIRSET = %s", tmps) == 1)
		dirset = strdup(tmps);
	else if(sscanf(line, "INTEREST = %s", tmps) == 1)
		intrfile = strdup(tmps);
	else if(sscanf(line, "ORDERIZE = %s", tmps) == 1)
		orderize = strdup(tmps);
	else if(sscanf(line, "FUNC = %[^\n]", tmps) == 1)	{
		if(fcn)
			free(fcn);
		fcn = strdup(tmps);
		if(dll_name)	{
			free(dll_name);
			dll_name = 0;
			gpk_warn("both a DLL and FUNC: using FUNC", 0);
			}
		}
	else if((tmp=sscanf(line, "DLL = %s %[^\n]", tmps2, tmps)) >= 1) {
		if(fcn)	{
			free(fcn);
			gpk_warn("multiple DLLs or FUNCs: using last DLL", 0);
			}
		if(dll_name)	free(dll_name);
		dll_name = strdup(tmps2);
		fcn = tmp>1 ? strdup(tmps) : strdup("");
		}
	else if(sscanf(line, "DLL_INIT = %[^\n]", tmps) == 1)	{
		dll_init = strdup(tmps);
		}
	else if(sscanf(line, "p%d = %lf +- %lf", &tmp, &td1, &td2) == 3)	{
		p_i_error_check(n, tmp, __LINE__);
		D fprintf(stderr, "setting %d to %g +- %g\n", tmp, td1, td2);
		pc[tmp] = td1;
		p[(tmp+n-firstprm)%n][tmp] = td2;
		p[n][tmp] -= td2;
		}
	else if(sscanf(line, "p%d = %lf", &tmp, &td1) == 2)	{
		p_i_error_check(n, tmp, __LINE__);
		pc[tmp] = td1;
		}
	else	{
		print_help();
		gpk_err("can't parse input line:", line);
		}
	}

 fclose(inp);


 {
 int i;
 for(i=0; i<nproc; i++)
	if(processor_label[i] && processor_label[i][0]!='\0')
		break;	/* There is at least one label */
 if(i < nproc)	{
	int j;
	for(j=0; j<nproc; j++)
		if(!processor_label[j] || processor_label[j][0]=='\0')	{
			fprintf(stderr, "ERR: opt: all processors must be labeled if any are. "
					"Label on %d, but none on %d.\n", i, j);
			exit(3);
			}
	}
 }

 if(n == 0)	{
	fprintf(stderr, "N not set in control file\n");
	print_help();
	exit(1);
	}

 if(dll_init && !dll_name)
	gpk_err("DLL_INIT without DLL specified", 0);

 if(nproc==1 ? timeout<=0 : timeout<0)
	timeout = Longtime;

 if(dirset)
	read_dirset(p, n, dirset, max(T0, ftol));

 if(timeout > 0)	{
	int i;
	for(i=0;i<nproc;i++)	{
		time_statistic[i].add(timeout);
		time_statistic[i].add(timeout);
		}
	}

 assert(double2test(p) == 0);
 assert(double1test(pc) == 0);
D fprintf(stderr, "setting up p:\n");
{
 int i;
 for(i=0;i<n+1;i++)	{
	int j;
	for(j=0;j<n;j++)	{
		if(!finite(p[i][j]) || !finite(pc[j]))	{
			char buf[250];
			sprintf(buf, "p[%d][%d]=%g, or pc[%d]=%g", i, j, p[i][j], j, pc[j]);
			gpk_err("non-finite input parameter or range:", buf);
			}
		p[i][j] = p[i][j]*sqrt(n) + pc[j];
		assert(finite(p[i][j]));
		}
	}
}

 assert(double2test(p) == 0);

 if(orderize)
	do_orderize(p, pc, n, orderize, firstprm);

 double1free(pc);

 if(shuffle)
	do_shuffle(p, n);
 if(intrfile)
	interest = do_interest(intrfile, n);

 if(!isnan(T0) && T0<0)	{
	char buf[100];
	sprintf(buf, "T0=%g", T0);
	gpk_err("Anneal requested with silly T0 ", buf);
	}
 else if(!finite(T0))
	T0 = 0;

 if(dll_name)	{
	assert(dlsym(RTLD_DEFAULT, "_etext") == &_etext); /* Sanity check. */
	dll_handle = dlopen(dll_name, RTLD_LAZY);
	if(!dll_handle)	{
		char buf[512];
		sprintf(buf, "can't do delayed link of %s:", dll_name);
		gpk_err(buf, dlerror());
		}

	dll_pointer = (chi_squared_t)dlsym(dll_handle, "chisq");
	if(!dll_pointer)	{
		fprintf(stderr, "ERR: opt: can't load chisq() from DLL %s: %s\n", dll_name, dlerror());
		exit(1);
		}

	int (*init)(char *C*) = (int (*)(char *C*))dlsym(dll_handle, "init");
	if(init)	{
		fprintf(stderr, "INFO: initializing DLL %s\n", dll_name);
		char **call = new char* [(dll_init?strlen(dll_init)/2:0)+2 + n + EXPSZ];
		build_arglist(call, dll_init?dll_init:"", n, -1, -1);
		fprintf(stderr, "Built call.\n");
		C int retval = (*init)(call);
		free_call(call);
		assert(retval);
		}
	}

 signal(SIGTERM, killer);
 signal(SIGQUIT, killer);
 signal(SIGINT, killer);
 if(signal(SIGHUP, killer) == SIG_IGN)
	signal(SIGHUP, SIG_IGN);

 assert(double2test(p) == 0);
 D fprintf(stderr, "go!\n");
 double **q;
 q = amoeba_x_allfree(p, n, ftol,
			dll_pointer ? funk_dll : funk_exec,
			itmax, (void *)fcn, 0, nproc, T0,
			interest);

 if(!q)
	gpk_err("Fit failed", 0);
 if(dll_handle)	{
	int (*f)() = (int (*)())dlsym(dll_handle, "done");
	if(f)
		(*f)();
	dlclose(dll_handle);
	}

 double *qs = amoeba_spread((C double *C*)q, n);

 fprintf(stderr, "writing output file...\n");
{
 int i;
 printf("N=%d\n", n);
 printf("TOL=%g\n", ftol);
 if(!isnan(T0) && T0>0)
	printf("T0=%g\n", T0);

 if(itmax % n == 0)
	printf("ITMAX=%d*N\n", itmax/n);
 else
	printf("ITMAX=%d\n", itmax);

 if(dirset)
	printf("DIRSET=%s\n", dirset);
 if(intrfile)
	printf("INTEREST=%s\n", intrfile);
 if(orderize)
	printf("ORDERIZE=%s\n", orderize);

 if(shuffle)
	printf("FIRSTPRM=SHUFFLE\n");
 else if(firstprm > 0)
	printf("# FIRSTPRM=%d   # (or maybe RANDOM)\n", firstprm);


 if(nproc > 1)
 	printf("NPROC=%d\n", nproc);
 for(i=0;i<nproc;i++)
	if(processor_label[i])
		printf("LABEL%1d=%s\n", i, processor_label[i]);

 if(dll_handle)
	printf("DLL=%s %s\n", dll_name, fcn);
 else	printf("FUNC=%s\n", fcn);

 printf("\n");
 for(i=0;i<n;i++)
	printf("p%1d = %g +- %g\n", i, p[0][i], qs[i]);
 fflush(stdout);
}
 fprintf(stderr, "done.\n");
 exit(0);
 return 0;
}

