# -*- coding: utf-8 -*-
#! python
from __future__ import with_statement

"""This script reads in the fiat file with data, finds all manually segmented files and creates symlinks
to mfcc and lab files for training data set for HTK. Can be run with mfcc.bash
Arguments: name_of_mfcc_file (no extension) metadata_file
The metadata file is in fiat format; each line describes one utterance.
"""

import os
import sys
import dummy_threading as _threading

from gmisclib import die
from gmisclib import fiatio
from gmisclib import dictops
from gmisclib import gpkmisc
from gmisclib import g_encode
# from gmisclib import threaded_io
from gmisclib import wavesurfer_lab as WS

from select_paragraphs import misc

sys.path.insert(0, 'lib')
import process_phones as PP
import HTK_write as HW
import select_utterances as SU
import generate as GE

RESP = '/proj/aesop/proc'
MFCC = 'mfcc'
LAB = 'lab'
GRAM = 'gram'
# Drop English ASR sentences and also group!=1 in English
# This helps balance the corpus
Vfile = '/proj/aesop/select_paragraphs/List_of_phones.txt'
TONES = ('_0', '_1', '_2', '_3', '_4')
MIN_PHONES_PER_UTTERANCE = 4






def run(dataid, metadatafilenames, mapname, root, whichpass):
	phonology = misc.read_phone_props(Vfile)
	pmap = PP.make_pmap(phonology, mapname)

	datalist = []
	for mdfn in metadatafilenames:
		metadatafile = open(mdfn, 'r')
		dbdir = os.path.dirname(mdfn)
		hdr, data, comments = fiatio.read(metadatafile)
		metadatafile.close()
		datalist.extend( GE.add_to_dict(data, metadatafile=mdfn, dbdir=dbdir) )
	assert len(datalist) > 0

	gpkmisc.makedirs(os.path.join(root, LAB))
	gpkmisc.makedirs(os.path.join(root, MFCC))
	gpkmisc.makedirs(os.path.join(root, GRAM))
	fnmap = fiatio.writer(open(os.path.join(root, 'metadata.fiat'), 'w'))
	fnmap.comment("This is a secondary metadata file, created during the ASR operation.")
	fnmap.comment("It is a copy of the following files, with added info on names used by the ASR system.")
	for mdfn in metadatafilenames:
		fnmap.comment(mdfn)
	# random.shuffle(datalist)
	odatalist = []
	for info in datalist:
		try:
			phonelists, priority = PP.prepare_phones_aesop_main(info, pmap, phonology, whichpass==0,
									whichpass!=0, pp=PP.PP[mapname])
		except IOError, x:
			die.warn("Cannot read %s/%s/%s: %s" % (info['dbdir'], info['d'], info['f'], x))
			continue
		except PP.BadPhonesError, x:
			die.warn("%s in %s/%s/%s" % (x, info['dbdir'], info['d'], info['f']))
			continue
		info['_priority'] = priority
		# Be aware that prepare_phones() can return more than one
		# alternative transcription, such as when there are several
		# human transcribers.   In such a case, we use them all.
		for (i,phones) in enumerate(phonelists):
			tmp = info.copy()
			tmp['_phones'] = phones
			tmp['_uid'] = '%s,%s,%d' % (info['d'], info['f'], i)
			if SU.selected(tmp, whichpass):
				odatalist.append(tmp)
	assert odatalist
	odatalist, allphones = PP.merge_rare_phones(odatalist, 20/(1 + 4*(whichpass==0)))
	HW.write_phonelist(root, allphones)

	# Write the human stuff (zero priority) first.
	# Then the stuff with checked transcriptions, finally the machine-generated
	# transcriptions.
	odatalist.sort(key=lambda x: x['_priority']-1.0/len(x['_phones']))

	nn = HW.make_nicknames([info['_uid'] for info in odatalist])
	for info in odatalist:
		info['_uid'] = nn[info['_uid']]
	tlf = open(os.path.join(root, 'trainlist'), 'w')
	# tpl = threaded_io.Lock()
	tpl = _threading.Lock()
	# tp = threaded_io.Thread_poolW(5)
	for info in odatalist:
		# tp.start(do_output, args=(info, root, dataid, whichpass==0, tlf, fnmap, tpl))
		GE.do_output(info, root, dataid, whichpass, tlf, fnmap, tpl, pmap)
	# tp.join()




if __name__ == '__main__':
	dataid = sys.argv[1]
	root = sys.argv[2]
	metadatafilenames = sys.argv[5:]
	# run(mfcc, metadatafilenames, 'VorC', root)
	whichpass = int(sys.argv[3])
	modeltype = sys.argv[4]
	run(dataid, metadatafilenames, modeltype, root, whichpass=whichpass)
