"use strict";
/*
speed/memory performance  is not important
ease of use is important
 */

/*
Purpose is to execute model code against the database to perform DDL operations.
To automate the DDL maintenance.
 */

/*

#Guidelines
The following is some guidelines.

#General
Updating a production  SQL database is always a challenge for the following reasons:
 * Data integrity is the number one concern.
 * Referral integrity, Stored procedures, triggers and views depend on the table structures,
this makes it almost impossible sometimes to change a table. The intricacy  mean a programmer
has to plan a upgrade path and implement a process.
 * often this this would involve
1) taking the database off-line (single user mode).
2) deleting some or all triggers, stored procedures, views and referral integrity.
4) altering simple columns.
5) creating new columns, copying / transforming data, drop the old columns, rename the new columns back.
6) recreating triggers, stored procedures, views and referral integrity.
 * this process is much to delicate to automate without extensive testing.
 * for this reason organisations often don't allow programmers to make such changes and rather
have (highly paid)dedicated database administrators for such tasks.


Updating a development database is fortunately not so bad because data integrity is not a concern, but development time is.
so it is ok to "trust" a automated process on a dev database.

So on a production database the compiler won't attempt any DDL updates, it trusts the models match the actual database.

During early stages of development it useful not to have to much referral integrity, as it makes such updates difficult,
add it in during later stages of development.
(it would be nice in this updater code to to be able to switch of referral integrity.)

#updater features
##during application compiling.
 * can do simple table changes(or create) according to model files.
 * can add indexes, triggers and stored procedures.
 * can populate demo data according to simple scripts.

##command line
 * can export a schema update script from the development database.
 * can import the schema update script to another database.


#Workflow
 * create database - command line   sql-mvc database [-a application] new_name
 * update/add model files.
 * update / add application pages.
 * the compiler will see the model changes and update the database schema.
 * test the application.
 * export the schema update script, and review, possibly keep in GIT for version control.
 * clone the production box including database to a sandbox.
 * update the sandbox to the new upp and test.
 * clone the production box including database to a backup / standby.
 * optionally take the production box off-line
 * update the production box to the new app and test.
 * bring the production box on-line

 */

/*
Ported from UpdateFDB.py
#Warnings
#   Don't run this against your live databases without extensive testing for suitability
#   the effects on tables with referral integrity is unknown


#NOTES
#   much of this process has been inspired by Flamerobin, especially DDL extract and how they create empty procedures before the tables.
#   Supports firebird 2.0 and 2.5 (CREATE OR ALTER VIEW)
#
#   The parsing is quite simple and relies on expected behaviour
#   we assume the ddl file was generated by Flamerobin (extract ddl).
#   we want to keep comments that form part of procedures - thus we cannot just strip all comments.
#   we presume ; , SET TERM ; ^ and SET TERM ^ ; is not going to appear somewhere in comments, and will always be on the start of the line.
#   we presume that terminating ; will always be at the end of a line and no joining of lines or comments after will occur.
#   we presume the table fields will always be on one and only one line and the terminator ); will always be on its own line
#   The process will not delete tables or fields that no longer exist in the source

#Improvements
#   for more detailed parsing: http://pyparsing.wikispaces.com/
#   Referral integrity?
#   Output info on tables and fields that could be deleted manually

#
#Issues
# The FDB driver seems to be stuck in dialect 2, no matter that you specify 1 or 3 this results in:
#   Date fields throw an error: DATE must be changed to TIMESTAMP - even with "cast (tsf as DATE)" which is a real big problem- code has to be rewritten with Extract
#   convert as:  select cast(extract(day from cast('now' as timestamp))||'.'||extract(month from cast('now' as timestamp))||'.'||extract(year from cast('now' as timestamp)) as timestamp) from RDB$DATABASE
#   Quoted identifiers  throw an error  : a string constant is delimited by double quotes, -104, 335544569

 */
var fs = require('fs');
var deepcopy = require('deepcopy');
var extend = require('node.extend');
var deasync = require('deasync'); var deasync_const=15;

exports.module_name = 'db_fb_updater.js';

var countLines = function (str) {
	return (str.match(/\n/g) || []).length;
};


function block_comment_suppress(inputs) {
	//replaces / * * / style comments with spaces (keeps line an col numbering consistent compared to the source)
	var rep_chr = " ";
	while (1) {
		var m = inputs.match(/([\S\s]*)(\/\*[\S\s]*\*\/)([\S\s]*)/i);
		if (m === null)
			break;
		//console.log('comment_replace : ',m[2]);

		var cr = m[2].replace(/\S/g, rep_chr);
		//console.log('comment_replace cr: ',cr);
		inputs = m[1] + cr + m[3];
	}

	//console.log('done ',inputs);
	return inputs;
}

function inline_comment_suppress(inputs) {
	//replaces --style comments with spaces (keeps line an col numbering consistent compared to the source)
    inputs = inputs.replace(/--[\s\S]*?$/gmi,'');
	return inputs;
}
function comment_suppress(inputs) {
   return inline_comment_suppress(block_comment_suppress(inputs));
}

var zx = require('../compiler/zx.js');

function dll_blocks_seperate_term(inputs, src_obj) { //splits the input into blocks that are in set term and blocks out side

	//function interpret_sorting_order(block){    }
	//console.log('dll_blocks_seperate_term start  A :', inputs, ':z');//.substring(0,20));
	var open_term = ";";
	var blocks = [];
	var inputsx='';

	if (zx.fb25) { 
	} else if (zx.mysql57) {	
		//replace mysql DELIMITER ;;    and DELIMITER ;  with firebird DELIMITERs
		inputs = inputs.replace(/DELIMITER\s*;;/i, "SET TERM ^ ; ");
		inputs = inputs.replace(/DELIMITER\s*;/i, "SET TERM ; ^ ");
	} else if (zx.mssql12) {	
	} else throw new Error("dialect code missing");

	while (inputs !== '') {
		//console.log('dll_blocks_seperate_term :', inputs);
		
		//remove leading comments
		do {
			inputsx=inputs;
		    inputs=inputsx.replace(/(\s*\/\*[\s\S]*?\*\/|([^:]|^)\/\/.*$)/,"");
		} while (inputsx!=inputs);
		
		var regs;
	    regs= '([\\S\\s]*?)set\\s+term\\s+(.)\\s+\\' + open_term + '([\\S\\s]*)';		
		//  ([\S\s]*)set\s+term\s+(.)\s+;([\S\s]*)
		var re = new RegExp(regs, 'i');
		var m = inputs.match(re);
        //console.log('dll_blocks_seperate_term split  B :', m, '<<<<<y');
		if (!m)
			m = [0, inputs, ';', ''];

		var l = countLines(m[1]);
		inputs = m[3];

		//remove xtra lines and terminator

		var t = m[1].replace(/\s+$/g, '');

		//console.log('remove leading lines  a :', m[1].substring(0,20));
		//console.log('remove leading lines  b :', t.substring(0,20));

		if (t.slice(-1) === open_term)
			t = t.slice(0, -1);

		//remove leading lines
		//console.log('remove leading lines b4 :',open_term, m[1].substring(0,20),'...',m[1].slice(-20));
		//var n = t.match(/(\s*)([\S\s]+)/i) || [0,  '',t, '']; ;
        //var n = [0, '', t, ''];
		//console.log('remove leading lines   :', n);
		var o = 0;//countLines(n[1]);

		if (open_term == ';') { //split into ;
			var statements = zx.splitNoParen(t, ";") || [];
			//console.log('statements   :', open_term,statements.length,n[2].substring(0,30));
			//console.log('statements   :', open_term,statements.length,t);
            //console.log('statements []  :', open_term,statements);
			//console.log('last statement   :', statements.slice(-1));

			for (var indx = 0; indx < statements.length; indx++) {
				var statement = statements[indx];
				//console.log('    statement   :', indx, zx.show_longstring(statement));
				//if (/*indx==1&&*/statement.match('PK_CACHE2')) process.exit(2);
				var ms = statement.match(/(\s*)([\S\s]+)/i); //remove leading lines
				//console.log('    statement   :', ms);
				if ((ms === null) || (ms === undefined)) {}
				else {
					l = countLines(ms[2]);
					o += countLines(ms[1]);
					var block = {
						t : ';',
						indx:indx,
						q : ms[2] + ';\n\n',						
						src : {
							l : l,
							o : o,
							src_obj : src_obj //lots of lines can share the same source file object
						}
					};
					//interpret_sorting_order(block);
					blocks.push(block);
				}

				o = 0;

			}
		} else {
			//console.log('bblock   :', open_term,n[2].substring(0,30));
			var bblock = {
				t : open_term,
				q : t, //driver dont need set term //"\nSET TERM ^ ;\n"+ n[2] + "\nSET TERM ; ^\n\n",
				src : {
					l : l - o,
					o : o,
					src_obj : src_obj //lots of lines can share the same source file object
				}
			};
			//interpret_sorting_order(block);
			blocks.push(bblock);
		}

		open_term = m[2];
	}
	//console.log('dll_blocks_seperate_term  :', blocks);
	return blocks;
}

var exec_qry_cache = {};
var exec_qry = function (cx, qrys) {
	if (exec_qry_cache[qrys]) {
		//console.log("exec_qry_cache: " ,qrys.substring(0,80));
		return;	
	}
	//console.log("exec_qry: " ,qrys.substring(0,80));
	exec_qry_cache[qrys] = true;
	exec_qry_non_cached(cx, qrys);
}

var exec_qry_non_cached = function (cx, qrys) {
	
	if (zx.fb25) { 
	} else if (zx.mysql57) {
		qrys = qrys.replace(/--:/g, "-- :");
		qrys = qrys.replace(/cast\s*\(\s*'now'\s+as\s+timestamp\s*\)/gi, " NOW() "); //also check compile.js:687
	} else if (zx.mssql12) { 		
	} else throw new Error("dialect code missing");
	

	exports.write_log.push(qrys);
	//fs.writeFileSync(zx.output_folder + 'build.sql', exports.write_log.join(''));
	//console.log('exports.write_log.push  :', qrys);
    //console.log('exec : ', qrys);
	//console.log('exec : ', qrys.substr(0,40));
	//console.log('exports.write_log.push  :', cx.zx.line_obj.srcinfo );

	if (cx.zx.config.db.schema_mode !== "slave") {
		//console.log('exec_qry_cb.sync  :', qrys );
		cx.zx.dbu.exec_qry_cb(cx, "exec_qry fn", qrys, cx.zx.line_obj);
	}
	delete cx.expect;
};




var dataset = function (zx, qrys) {//could use the one from sql_utils
    //console.log("dataset:" ,qrys);
	var result = zx.dbu.fetch_dataset(zx , "updater dataset", qrys);
	//console.log("dataset:" ,result);
	return result;
};
var singleton = function (zx, field, qrys,debug) {//could use the one from sql_utils
    //console.log("singleton qrys:",debug ,qrys);
	var result = zx.dbu.fetch_dataset(zx, "singleton fetch_dataset :", qrys, 0);
    //console.log("singleton res:" ,typeof result, result);
	if (debug) 
		console.log("singleton :",result,qrys);
	if (result[0] === undefined) {
		console.log("singleton:", typeof result, result);
	}
	
	//console.log("singleton res fields:" ,fields[0].name);
	//console.log("singleton res x:" ,typeof result[0], result[0]['count(*)']);
	//console.log("singleton res JSON:" ,JSON.stringify(result,null,4));
	
	//console.log("singleton res z:" ,typeof result[0].RowDataPacket, result[0].RowDataPacket);
	//console.log("singleton res z:" ,result[0].['count(*)']);
	
	//if (result[0].RowDataPacket[field] !== undefined) {	}
	
	if (zx.mssql12) {
		if (debug) console.log("singleton mssql12:",result[0] ,result,qrys);
		return result[0];
	}
	
	if (result[0] == undefined) {
		console.log("singleton empty result:",result,qrys);
		return '';
	}
	if (result[0][field] !== undefined) {
		//console.log("singleton:" ,result[0][field].low_)
		if (result[0][field].low_ === undefined)
			return result[0][field];
		return result[0][field].low_ + (result[0][field].high_ * 65536 * 65536);
	} else {
		console.log('singleton qq: ', qrys);
		console.trace("singleton unknown field :", field, result);		
		throw new Error("singleton unknown field B:"+ field); 
		return '';
	}
		
};
var checkTable = function (zx, name) {
	if (zx.fb25)
        return singleton(zx, "COUNT", "select count(*) from rdb$relations where rdb$relation_name ='" + name.toUpperCase() + "' ;");	
	if (zx.mysql57)
        return singleton(zx, "count(*)", "select count(*) from information_schema.tables where table_schema = '"+zx.conf.db.database_filename+"' AND table_name = '" + name.toUpperCase() + "' AND TABLE_TYPE='BASE TABLE' ;");
    if (zx.mssql12)
        return singleton(zx, "count(*)", "select count(*) from information_schema.tables where TABLE_CATALOG = '"+zx.conf.db.database_filename+"' AND TABLE_SCHEMA = '" +zx.conf.db.database_schema+"' AND TABLE_NAME = '" + name.toUpperCase() + "' ;");

	throw new Error("dialect code missing");
};
var checkView = function (zx, name) {
	if (zx.fb25)
	    return singleton(zx, "COUNT", "select count(*) from rdb$relations where rdb$relation_name ='" + name + "' AND (rdb$view_blr IS NOT NULL);");		
	if (zx.mysql57)
        return singleton(zx, "count(*)", "select count(*) from information_schema.tables where table_schema = '"+zx.conf.db.database_filename+"' AND table_name = '" + name.toUpperCase() + "' AND TABLE_TYPE='VIEW' ;");
//	if (zx.mssql12)	
	throw new Error("dialect code missing");
};

var checkGenerator_cache = {};
var checkGenerator = function (zx, name,debug) {
	var sg=checkGenerator_cache[name];
	//console.log('checkGenerator : ', sg);
	if (!sg) {
		//console.log('checkGenerator non: ', sg);
		sg=checkGenerator_non_cached(zx, name,debug);
		checkGenerator_cache[name]=sg;
	}
	return sg;
}
var checkGenerator_non_cached = function (zx, name,debug) {
	//CREATE SEQUENCE IF NOT EXISTS myschema.myseq;  Postgres 9.5+
	if (zx.fb25)
	    return singleton(zx, "COUNT", "SELECT count(*) FROM RDB$GENERATORS  where RDB$GENERATOR_NAME='" + name + "' ;",debug);	
	if (zx.mysql57)
        return singleton(zx, "count(*)", "select count(*) from information_schema.tables where table_schema = '"+zx.conf.db.database_filename+"' AND table_name = '" + name.toUpperCase() + "' AND TABLE_TYPE='BASE TABLE' ;",debug);
	if (zx.mssql12)
        return singleton(zx, "count(*)", "SELECT count(*) FROM sys.sequences WHERE name = '" + name.toUpperCase() + "';",debug);	
	throw new Error("dialect code missing");
};


var getGenerator_cache = {};
var getGenerator = function (zx, name, increment,debug) {
	var sg=getGenerator_cache[name];
	//console.log('getGenerator_cache : ', sg);
	if (!sg) {
		//console.log('getGenerator_cache non: ', sg);
		sg=getGenerator_non_cached(zx, name, increment,debug);
		getGenerator_cache[name]=sg;
	}
	return sg;
}
var getGenerator_non_cached = function (zx, name, increment,debug) {
 //console.log("getGenerator:", name,zx.fb25);
	if (zx.fb25) { 
		return singleton(zx, "GEN_ID", "SELECT GEN_ID( " + name + "," + increment + " ) FROM RDB$DATABASE;",debug);
	} else if (zx.mysql57) { 
		return singleton(zx, "AUTO_INCREMENT", "select AUTO_INCREMENT from information_schema.tables where table_schema = '"+zx.conf.db.database_filename+"' AND table_name = '" + name.toUpperCase() + "' AND TABLE_TYPE='BASE TABLE' ;",debug);
	} else if (zx.mssql12) { 
        return singleton(zx, "current_value", "SELECT current_value FROM sys.sequences WHERE name = '" + name.toUpperCase() + "';",debug);	
	} else throw new Error("dialect code missing");	
};
var dropTriggers = function (zx) {
	var t = dataset(zx, "select rdb$trigger_name from rdb$triggers where (rdb$system_flag = 0 or rdb$system_flag is null)");
	//console.log("dropTriggers:",t);
	t.forEach(function (rec) {
		var sql = "drop trigger " + rec.rdb$trigger_name;
		console.log("dropTriggers forEach:", rec.rdb$trigger_name);
		exec_qry({
			zx : zx
		}, sql);
	});
	return t.length;
};

var dropProcedures = function (zx) {
	//remove the code from procedure - alternative to drop/create empty which has a dependency issue
	var count = 0;
	var totaldropped;
	//each loop any independent procs are dropped, thereby freeing dependencies of the other procs
	while (count++ < 10) //limit the loops in case a procedure cant be dropped
	{
		var t = dataset(zx, 'SELECT rdb$procedure_name FROM rdb$procedures ' +
				'WHERE rdb$system_flag IS NULL OR rdb$system_flag = 0;');
		console.log("DropProcedures :", t);
		if (totaldropped === null)
			totaldropped = t.length;
		if (t.length === 0)
			break;
		for (var indx = 0; indx < t.length; indx++) {
			var rec = t[indx];

			var sql = "drop procedure " + rec.rdb$procedure_name;
			console.log("DropProcedures forEach:", rec.rdb$procedure_name);

			exec_qry({
				zx : zx,
				expect : /335544673/
			}, sql);
		}
	}
	return totaldropped;

};
var dropDependecies = function (zx) {
	if (zx.config.db.schema_mode !== "master")
		return; //safeguard against deleting someone else's procedures
	//drop constraints // FOREIGN KEY Constraint
	dropTriggers(zx);
	dropProcedures(zx);

};

var mssql_alter_table_drop_default = function (schema,table,field) {
//https://www.oraylis.de/blog/how-to-delete-an-unnamed-default-constraint-in-an-sql-server-table	
	var qry = 
    "declare @cmd nvarchar(150) "+
"\r\nselect @cmd = "+
"\r\n   'ALTER TABLE ' + t2.name + '.' + t1.name + ' DROP CONSTRAINT ' + "+
"\r\n    t4.name from [sys].[tables] t1 join [sys].[schemas] t2 on t1.schema_id = t2.schema_id "+
"\r\n	join [sys].[all_columns] t3 on t1.object_id = t3.object_id "+
"\r\n	join [sys].[default_constraints] t4 on t1.object_id = t4.parent_object_id and t3.column_id = t4.parent_column_id "+
"\r\n	where t1.name = N'"+table+"'"+
"\r\n	and t2.name = N'"+schema+"'"+
"\r\n	and t3.name = N'"+field+"'"+
"\r\nif @cmd is not null exec @cmd ";
return qry;
}	

var CREATE_TABLE = function (zx, qrystr) {
	var cx = {
		zx : zx
	};
	var qryout='';
	var CREATE_GLOBAL_TEMPORARY_TABLE =0;
	var barestr = comment_suppress(qrystr); 
	var Table = barestr.match(/CREATE\s+TABLE\s+([\w$]+)(\s*\(\s)([\S\s]*)\)/i);
	if (!Table) {
		Table = barestr.match(/CREATE\s+GLOBAL\s+TEMPORARY\s+TABLE\s+([\w$]+)(\s*\(\s)([\S\s]*)\)/i);
		if (Table) CREATE_GLOBAL_TEMPORARY_TABLE =1;
	}
	if (!Table)
		{//did not understand try to create as is
        console.log('Could not undestand the table code - attempt to update as is');
		exec_qry(cx, qrystr);
		return "";
	} 
	barestr = Table[3];
    //onsole.log('Table re:', Table, barestr,' >',qrystr,'[',Table);
	Table = Table[1];

	
	var tableexists = checkTable(zx, Table);
	//console.log("tableexists:",tableexists,Table);//,"qrystr:",qrystr," barestr:",barestr);
	
	var fields = zx.splitNoParen(barestr, ',');
	var FieldNumber = 0;
	var newfield,defs;
	
	//console.log('CREATE_TABLE ',Table);
	//console.log('\r\n\r\n\r\n-------------------------------:','Table ',Table,' fields:',fields ,'\r\n\r\n\r\n-------------------------------');	
	
	

	//console.log('-------------------------------: fake_domains(empty on fb engine):\r',zx.sql.fake_domains );
	barestr = "";
	
	fields.forEach(function (field) {
		field = field.trim();
		var original_input_field = field;
		
		if (field.length>180) {
		//	zx.error.log_model_warning(zx, "Field definition seems very long - typical of a brace syntax error :" , field, zx.line_obj);		
		}
		
		//console.log('\r\n-------------------------------: fields:\r\n>',field,'<');
		if ((field !== ")" && field !== ";")) {
			FieldNumber = FieldNumber + 1;
			
			newfield = field.replace(/UNIQUE/i, " ");	
			var Unique=(newfield != field);
			field=newfield;
			
			newfield = field.replace(/NOT\s*NULL/i, " ");	
			var NOT_NULL=(newfield != field);
			field=newfield;
					
			newfield = field.replace(/AUTO_INCREMENT/i, " ");
			var AUTO_INCREMENT=(newfield != field);
			field=newfield;
			
			newfield = field.replace(/PRIMARY\s+KEY/i, " ");
			var PRIMARY_KEY=(newfield != field);
			field=newfield;
			
			field = field.replace(/MAXDATE/i, zx.config.db.sql_MAXDATE );		
			
			
			var default_value='';
			var default_set=0;
			var FFD = field.match(/default\s+(\S+)/i);
			if (FFD) {					
					//console.log('Table fields FFD:', field, FFD,default_value);
					default_value = FFD[1].trim();	
					default_set=1;
					field = field.replace(FFD[0], "");
					//console.log('Table fields FFDx:', field, default_value);					
				}
				
			if (zx.fb25) {
				//simple types
				field = field.replace(/TINYTEXT/i,   "VARCHAR(256)");	
				field = field.replace(/\sTEXT\s/i,   "BLOB SUB_TYPE 1");	
				field = field.replace(/MEDIUMTEXT/i, "BLOB SUB_TYPE 1");					
				field = field.replace(/LONGTEXT/i,   "BLOB SUB_TYPE 1");	
			} else if (zx.mysql57) {
				//simple fb to mysql types
				field = field.replace(/BLOB\s*SUB_TYPE\s*1/i,   "MEDIUMTEXT");
			    // blob is blob field = field.replace(/BLOB\s/i,                " VARBINARY(MAX)");
				if (default_value) {
					default_value = default_value.replace(/'now'/i, "CURRENT_TIMESTAMP");
					default_set=1;
					}
			} else if (zx.mssql12) {
				field = field.replace(/BLOB\s*SUB_TYPE\s*1/i,   " VARCHAR(MAX)");
				field = field.replace(/BLOB\s/i,                  " VARBINARY(MAX)");
				field = field.replace(/TIMESTAMP/i,   			"datetime");					
				if (default_value) {
					default_value = default_value.replace(/'now'/i, "CURRENT_TIMESTAMP");
					default_set=1;
					}				
			}  else throw new Error("dialect code missing");
			
			
			//console.log('--> ',field );	
			if (defs=field.match(/([`\w$]+)\s+([()\w$]+)/i)) {				
				// replace with fake domains				
				var fake_domain = zx.sql.fake_domains[defs[2].toUpperCase()];
				//console.log('defs  :',defs,' type :',defs[2], ' fd:',fake_domain );	
				if (fake_domain) {
				    //console.log('defs  name:',defs[1],' type :',defs[2], ' fd:',fake_domain );	
					defs[2] = fake_domain.type;
					if (fake_domain.set==1) {
					}
				}
			} else {
				//console.log('none matched ([`\w$]+)\s+([()\w$]+) ',field, ' in ' ,original_input_field);
				if (field.length>80) {
					zx.error.log_model_warning(zx, "Field definition seems very long - typical of a brace syntax error :" , field, zx.line_obj);
				} else {
					//console.log('none matched ',fields);
					zx.error.log_model_warning(zx, "none matched ([`\w$]+)\s+([()\w$]+) - typical of an earlier brace syntax error :" , field, zx.line_obj);
					//throw new Error("Syntax error");
					//...some syntax error
				}
			}
			
			
			if (zx.fb25) {					
				field = defs[1] + " " + defs[2]	+ (NOT_NULL?" NOT NULL":"") 
						+ (Unique?" UNIQUE":"") 
						+ (default_value?(" DEFAULT "+default_value):"")
						;
			} else if (zx.mysql57){				
				field = defs[1] + " " + defs[2]	+ (NOT_NULL?" NOT NULL":"") 
						+ (Unique?" UNIQUE":"") 
						+ (AUTO_INCREMENT?" AUTO_INCREMENT":"")  
						+ (PRIMARY_KEY?(" PRIMARY KEY "):"")
						+ (default_value?(" DEFAULT "+default_value):"")
						;				
				//if (Unique) bottom_of_create +=  '\r\n    UNIQUE('+defs[1]+')';
			} else if (zx.mssql12){	
				field = defs[1] + " " + defs[2]	+ (NOT_NULL?" NOT NULL":"") 
						+ (Unique?" UNIQUE":"") 
						+ (AUTO_INCREMENT?"  IDENTITY(1,1) ":"")  
						+ (PRIMARY_KEY?(" PRIMARY KEY "):"")
						+ (default_value?(" DEFAULT "+default_value):"")
						;			
			}  else throw new Error("dialect code missing");
				
			//block.name = 'TABLE_' + name[1];
			barestr = barestr + ",\r\n    " +field + ""; 
			//console.log('--> ',field );	
			
		}
	});
	
	barestr=barestr.substring(1);
	
	if (CREATE_GLOBAL_TEMPORARY_TABLE==0) qryout = "CREATE TABLE "+Table+" (\r\n" + barestr;
	if (CREATE_GLOBAL_TEMPORARY_TABLE==1) qryout = "CREATE GLOBAL TEMPORARY TABLE (\r\n" + barestr;		
	qryout = qryout + "\r\n)";
	
	//console.log('Table        :', Table,tableexists,qryout); 
	//console.log('Table qryout :\r\n', qryout );	
	//console.log('\r\n\r\n\r\nTable qryout -----------------------------:\r\n', qryout );
	//if (Table=="MAIL") process.exit(2);		
	qrystr = qryout;
	if (tableexists === 0) // create new table as is
	{
		var recodesql;
        console.log('Table does not exist  - attempt to create as defined '+Table);
		
		recodesql = qrystr.replace(/AUTO_INCREMENT/i, "/*AUTO INCREMENT*/");
		//console.log('**************************************************************AUTO_INCREMENT type check:', recodesql);
			
		var AUTO_INCREMENT=(recodesql != qrystr);		
		if (zx.fb25) {
				//if (fb30) //has syntax for auto creating				
				if (AUTO_INCREMENT) qrystr=recodesql;
				//add trigger code
				qrystr = qrystr.replace(/MEDIUMTEXT/i, "BLOB SUB_TYPE 1");				
		} else if (zx.mysql57){
			    qrystr = qrystr.replace(/BLOB\s+SUB_TYPE\s+1/i, "MEDIUMTEXT");
		} else if (zx.mssql12) {
			    qrystr = qrystr.replace(/BLOB\s+SUB_TYPE\s+1/i, "VARCHAR(MAX)");
		} else throw new Error("dialect code missing");
		
		exec_qry(cx, qrystr);
		return "";
	} else {

	

	fields = zx.splitNoParen(barestr, ',');
	fields.forEach(function (field) {
		field = field.trim();
		if ((field !== ")" && field !== ";")) {
			field = field.replace(/MAXDATE/i,   "'2030/01/01'");		
			//console.log('Table fields ---:', Table,'>',field,'<');
			FieldNumber = FieldNumber + 1;			
			cx.expect = zx.dbu.sqltype(zx,/unsuccessful metadata update/,/ER_DUP_FIELDNAME/,/is specified more than once/);
			exec_qry(cx, "ALTER TABLE " + Table + " ADD " + field +"; ");
            if (cx.zx.config.db.schema_alter_fields === "yes") { 
			
			//find and remove unique key			
			newfield = field.replace(/UNIQUE/i, " ");	
			var Unique=(newfield != field);
			field=newfield;
			
			newfield = field.replace(/NOT NULL/i, " ");	
			var NOT_NULL=(newfield != field);
			field=newfield;
			
			//todo mysql update  Unique and NOT_NULL only if wrong
			
			newfield = field.replace(/AUTO_INCREMENT/i, "/*AUTO INCREMENT*/");
			//console.log('**************************************************************AUTO_INCREMENT type check:', newfield);
			
			var AUTO_INCREMENT=(newfield != field);
			field=newfield;
			if ((zx.fb25)&&(AUTO_INCREMENT)){
				//if (fb30) //has syntax for auto creating
				//add trigger code
				
			}
			
			//console.log('Blob type check:', field);
			if (field.match(/\sblob\s*/i) 
				|| field.match(/\sCOMPUTED BY\s*/i)
			    || field.match(/\sMEDIUMTEXT\s*/i)
			    || field.match(/\sLONGTEXT\s*/i)
				|| AUTO_INCREMENT
				) {
				//Blobs cant be altered
				//Computed by gets dropped as part of cleanup ??WTF
				//console.log('Blob type has no default:', field);
				
				
			} else {

				var FFD = field.match(/(\S+)\s+(\S+)\s+(default)?\s?(not\s+null)?\s?(.*)/i);
				//console.log('Table fields FFD:', Table,'>',field,'<', FFD);
				if (FFD) { //with default set
					{
					var FieldName = FFD[1].trim(),
					FieldType = FFD[2].trim(),
					Default = FFD[5].trim();
					if (FFD[4])
						console.log('WARN: Cannot update "NOT NULL" property for :', Table + '.' + FieldName);
					
						if (zx.mssql12) {
							//TODO
							console.log('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX mssql12 ignoring default attr :', Default,":");
						} else {					
					
					
                    	
					cx.expect = zx.dbu.sqltype(zx,/unsuccessful metadata update/,/ER_DUP_FIELDNAME/,/is specified more than once/);
					if (zx.fb25) { 
						exec_qry(cx, "ALTER TABLE " + Table + " alter " + FieldName + " TYPE " + FieldType+"; "); 
					} else if (zx.mysql57) { 
						exec_qry(cx, "ALTER TABLE " + Table + " MODIFY " + FieldName + "  " + FieldType+"; ");
					} else 
						throw new Error("dialect code missing");
					
					//console.log('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Default:', Default,":");
					if (Default !== "") {
                        //console.log('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Table fields FFD Default:', Table,' . ',FieldName,Default);
                        
                        cx.expect = /unsuccessful metadata update/; 
						if (zx.fb25) {
							exec_qry(cx, "ALTER TABLE " + Table + " Alter " + FieldName + " set DEFAULT " + Default+"; ");
						} else if (zx.mysql57) {
							exec_qry(cx, "ALTER TABLE " + Table + "   MODIFY column  " + FieldName + "  " + FieldType + " DEFAULT " + Default+"; ");
						}  else throw new Error("dialect code missing"); 
                        //exec_qry(cx, "ALTER TABLE " + Table + " Alter " + FieldName + " set " + Default);
						// updating the default before commit seems a problem ... this should be moved to phase 2
						//caused an error in carlton update ->    exec_qry("update "+Table +" set " + FieldName + "="+Default+" where " +FieldName + " is null ")
					} else {
                        //console.log('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Table fields FFD        :', Table,' . ',FieldName);// FFD);

						cx.expect = /unsuccessful metadata update/;
						if (zx.fb25) {  
							exec_qry(cx, "ALTER TABLE " + Table + " Alter " + FieldName + " DROP DEFAULT "+"; ");
						} else if (zx.mysql57)  {
							exec_qry(cx,  "ALTER TABLE " + Table + "   alter column  " + FieldName + " DROP DEFAULT "+"; ");
						}  else throw new Error("dialect code missing"); 
                        
					}
					}
				}                                            

				}else { //no default so drop any default that may exist
						
						var FFD = field.match(/(\S+)\s+(\S+)/i);
						if (FFD) {							
								var FieldName = FFD[1];
								//console.log('\n\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX  Field default not found :', field,FFD);
								//should be dropping

								cx.expect = /unsuccessful metadata update/;
								if (zx.fb25) {
									exec_qry(cx, "ALTER TABLE " + Table + " Alter " + FieldName + " DROP DEFAULT ;");
								} else if (zx.mysql57) {
									exec_qry(cx,  "ALTER TABLE " + Table + "   alter column  " + FieldName + " DROP DEFAULT ;");
								} else if (zx.mssql12) { 
								    exec_qry(cx, mssql_alter_table_drop_default(zx.conf.db.database_schema,Table,FieldName));
								} else throw new Error("dialect code missing");
								
							} else {
								console.log('\n\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX  Unknown syntax for update table fields :', field,FFD);
								process.exit(2);
							}
						
				}
			}
                    if (cx.zx.config.db.schema_reorder_fields === "yes") { 
                        cx.expect = /unsuccessful metadata update/; 
						exec_qry(cx, "ALTER TABLE " + Table + " ALTER " + FieldName + " POSITION " + FieldNumber+"; ");						
                    }
			
            }
		}
	});
	
	}

	return "";
};


	
exports.csvtojson = function (zx,csvStr) {
    var  data=[],done=false;
	
	//console.log('csvtojson:',csvStr);
	const csv=require('csvtojson')
	csv({noheader:true})
	.fromString(csvStr)
	.on('csv',(csvRow)=>{ // this func will be called 3 times
		//console.log('csvtojson row:',csvRow); // => [1,2,3] , [4,5,6]  , [7,8,9]
		data = data.concat(csvRow);
	})
	.on('done',()=>{
		//parsing finished
		done=true;
		//console.log('csvtojson done:',data);
	})	
	while (!done) {
		deasync.sleep(deasync_const);
	}
	//console.log('csvtojson return:',data);
    return data;
}   

	
exports.parse_insert_update = function (zx, qrystr,insertmatchingfield)
{ //will always return an object
	var result ={};
	var ins = qrystr.match(/insert\s+into\s*([\S$]+)\s*\(([\S\s]*)\)\s*values\s*\(([\S\s]*)\)/i);
	//console.log('insert_update_variation:',ins);
	if (ins) {
		result.tablename = ins[1];
		result.fields = exports.csvtojson(zx,ins[2].replace('\n','') );
		result.values = exports.csvtojson(zx,ins[3].replace('\n','') );
		result.matchingfield = exports.csvtojson(zx,insertmatchingfield.replace('\n','') );
		return result;
	}
	throw new Error("insert_update_variation invalid 'insert into' syntax "+qrystr);
	//return null;
}

	
exports.insert_update_variation = function (zx, qrystr,insertmatchingfield)
{
	if (zx.mssql12) {
		var ins = exports.parse_insert_update(zx, qrystr,insertmatchingfield);
		var deleterecord="";
		ins.matchingfield.forEach(function (field) {
			if (deleterecord!='') deleterecord = deleterecord + " and ";
			var i = ins.fields.indexOf(field);	
			if (i<0) throw new Error("insert_update_variation error "+ field + " not in list "+ins.fields);
			deleterecord = deleterecord + field + "=" + ins.values[i];			
		});
		
		deleterecord="delete from "+ins.tablename+" where " + deleterecord ;
		//console.log('insert_update_variation deleterecord mssql12:',deleterecord);
		return deleterecord;
	}
	
	
	if (zx.fb25) {
		return "update or " + qrystr + " matching(" + insertmatchingfield + ") ";
	} else if (zx.mysql57) {
		//console.log('insert_update_variation in:',qrystr);
		var ins = exports.parse_insert_update(zx, qrystr,insertmatchingfield);	
		
		var deleterecord="";
		ins.matchingfield.forEach(function (field) {
			if (deleterecord!='') deleterecord = deleterecord + " and ";
			var i = ins.fields.indexOf(field);	
			if (i<0) throw new Error("insert_update_variation error "+ field + " not in list "+ins.fields);
			deleterecord = deleterecord + field + "=" + ins.values[i];			
		});
		
		deleterecord="delete from "+ins.tablename+" where " + deleterecord ;
		//console.log('insert_update_variation deleterecord:',deleterecord);
		return deleterecord;
	} else throw new Error("dialect code missing");
	
	throw new Error("insert_update_variation For unknown database dialect");
}
				


exports.Prepare_DDL = function (zx, filename, inputsx, line_obj) {

	var inputs;
	if ((inputsx === null) || (inputsx === undefined))
		inputs = fs.readFileSync(filename, 'utf8');
	else
		inputs = inputsx;

	//inputs = comment_replace(inputs);
	//console.log('Prepare_DDL filename  :', filename);
	//console.log('Prepare_DDL len :', inputs.length);
	//console.log('Prepare_DDL  :', inputs);
	
	var LineNr = 1;
	var BlockNr = exports.blocks.length;
	var insertmatchingfield = '';
	var insertmatchingblock;
	var verbosity = 10;
    var block,bi;

	//exports.input_audit.push(inputsx);
	//exports.input_audit.push(JSON.stringify(line_obj,null,4));//.start_line + ' ' + line_obj.filename);
	var blocks = dll_blocks_seperate_term(inputs, line_obj);
	exports.input_audit.push(JSON.stringify(blocks, null, 4)); //.start_line + ' ' + line_obj.filename);

	//exports.show_DDL(zx,"Creating",blocks);	
	for (var ixx = 0, max = blocks.length; ixx < max; ixx += 1) {	
	    bi=ixx;
		block=blocks[bi];
		LineNr += block.src.o;
		BlockNr++;

		//add more context info to the block source
		if (block.method !== "DECLARE_PROCEDURE") //src line already set
			block.src.LineNr = LineNr;
		block.src.BlockNr = BlockNr;
		block.src.ModelBlocksNr=bi;

		//console.log('blocks.forEach  :', LineNr, bi);
		//if (verbosity > 2)
		//	console.log(" Preview DDL from line preview: ",  block.src.LineNr, ' for ', block.src.l + 1, ' lines');
		var name,qrystr = block.q;
		//console.log('blocks.forEach GENERATOR:',qrystr.match(/\s*CREATE\s+GENERATOR\s/i) );
		//statements that would be used in the make script
        //console.log('blocks.forEach subs:',qrystr.substring(0,50));


		//name=qrystr.match(/CREATE\s+TRIGGER\s+(\'*\"*[\w$]+\'*\"*)/i);
		//console.log('blocks.forEach trigger^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^:',name);
		
		//change the conditional creates to normal creates they will be converted back to idempotent procedures
		qrystr = qrystr.replace(/^CREATE\sOR\sALTER\s+PROCEDURE\s/i, "CREATE PROCEDURE ");
		qrystr = qrystr.replace(/^ALTER\s+PROCEDURE\s/i, "CREATE PROCEDURE ");
		block.method = "exec";
		block.qrystr = 'xxxxx';

		if (qrystr.match(/QUICC\s+BREAK/i)) {
			throw new Error("QUICC BREAK in source");
		
		} else if (qrystr.match(/SET\s+VERBOSITY\s/i)) {
			var mv = qrystr.match(/SET\s+VERBOSITY\s+([\w\$]+)/i);
			if (mv) {
				verbosity = +mv[1];
				qrystr = "";
			}
			block.order = 100;
		
		} else if (name=qrystr.match(/CREATE\s+ROLE\s+([\w$]+)/)) {
			block.name = 'ROLE_' + name[1];
			block.expect = /unsuccessful metadata update/;
			block.order = 200;
			} else if (name=qrystr.match(/CREATE\s+DOMAIN\s+([\w$]+)/)) {
			//console.log(" match DOMAIN:  ",name);	
			block.name = 'DOMAIN_' + name[1];
			block.expect = /unsuccessful metadata update/;
			block.order = 300;

			if (zx.fb25) { 
			} else if (zx.mysql57||zx.mssql12) {
				var default_val="''";
				var default_set=0;
				var cdinfo;
				
				var clr = qrystr.replace(/NOT\s+NULL/i, " ");
				var  NOT_NULL=(clr != qrystr);
				qrystr=clr;				
				
				cdinfo=qrystr.match(/CREATE\s+DOMAIN\s+([\w$]+)\s+as\s+([()\w$]+)\s*;/i);
				//console.log(" match DOMAIN in mysql:  ",cdinfo);						
				if (cdinfo===null) {						
					cdinfo=qrystr.match(/CREATE\s+DOMAIN\s+([\w$]+)\s+as\s+([()\w$]+)\s+default\s+([''\w$]+)\s*;/i);
					if (cdinfo!==null) {						
						//console.log(" match DOMAIN and default in mysql:  ",cdinfo);	
						default_val=cdinfo[3];default_set=1;
						}
				}
				
				
				if (cdinfo===null) {						
					console.log("fake_domain fail:  ",qrystr);
					process.exit(2);
				}
				//console.log(" mysql DOMAIN: ",cdinfo[1],' type:',cdinfo[2],' default:',default_set,' v:',default_val);		
				
				zx.sql.fake_domains[cdinfo[1].toUpperCase()] = { type:cdinfo[2], set:default_set, value:default_val };
				
				//console.log("zx.fake_domains:  ",zx.sql.fake_domains);
				block.method = "bypass";
				//process.exit(2);
				}  else throw new Error("dialect code missing");
			
			
		} else if (name=qrystr.match(/DECLARE\s+EXTERNAL\s+FUNCTION\s+([\w$]+)/)) {
			block.name = 'FUNCTION_' + name[1];
			block.expect = /unsuccessful metadata update/;
			block.order = 400;
		} else if (name=qrystr.match(/CREATE\s+GENERATOR\s+([\w$]+)/i)) {
			var gen_name =   name[1];
			if (zx.mysql57) gen_name +=	"_GENERATOR";				        
			block.name = 'GENERATOR_' + gen_name;
			block.expect = /unsuccessful metadata update/;
			if (zx.fb25) { 
				if (checkGenerator(zx, gen_name)) {
					//console.log(" checkGenerator:  exits");
					qrystr = "";
					}
			} else if (zx.mysql57) { 
			    var cg=checkGenerator(zx, gen_name,0);
				//console.log("CREATE GENERATOR mysql57: ",cg,gen_name," ");
				if (cg=='') cg=0;
				if (cg>0) qrystr = "";
				else qrystr = "CREATE TABLE "+gen_name+"(id INT AUTO_INCREMENT PRIMARY KEY, x int);";
				//console.log("CREATE GENERATOR cg: ",cg,qrystr," ");
				// throw new Error("dialect code missing");
			} else if (zx.mssql12) { 
				qrystr = "";
			} else throw new Error("dialect code missing");
			//console.log("CREATE GENERATOR : ",qrystr," ");
			block.order = 500;
		} else if (name=qrystr.match(/SET\s+GENERATOR\s+([\w\$]+)\sTO\s([\w\$]+)/i)) {
			block.name = 'SETGENERATOR_' + name[1];			
			
			var gen_name =   name[1];
			if (zx.mysql57) gen_name +=	"_GENERATOR";
			//console.log("setGenerator  ",name, 'as ',gen_name);
			var gv=0,cg=checkGenerator(zx, gen_name,0);
			if (cg) gv = getGenerator(zx, gen_name, 0, 0);			
			//console.log("check setGenerator ",name[1],' exists :',cg ," is set to ",gv);
			if (gv>0) { //ignore if already set
				qrystr = "";
				//console.log("setGenerator ",name[1]," already set to ",gv);
			}
			else {
				if (zx.fb25) { 
					// allow the set value though as is	
					//console.log("setGenerator as ",qrystr);					
				} else if (zx.mysql57) { 
					qrystr = "ALTER TABLE "+gen_name+" AUTO_INCREMENT = "+name[2]+" ;";
				} else if (zx.mssql12) { 
					qrystr = "CREATE SEQUENCE "+gen_name+" AS INT START WITH "+name[2]+" INCREMENT BY 1;";
				} else throw new Error("dialect code missing");
			}		    
			block.order = 600;
		} else if (name=qrystr.match(/CREATE\s+SEQUENCE\s+([\w$]+)/i)) {
			if (zx.mssql12) {
				qrystr = "";
			} else {
			
			block.name = 'SEQUENCE_' + name[1];
			block.expect = /unsuccessful metadata update/;
			var mc = qrystr.match(/CREATE\s+SEQUENCE\s([\w\$]+)/i);
			if (mc && checkGenerator(zx, mc[1]))
				qrystr = "";
			// else execute as is
			}
			block.order = 500;
		} else if (name=qrystr.match(/ALTER\s+SEQUENCE\s/i)) {
			block.name = 'ALTERSEQUENCE_' + name[1];
			var msgg = qrystr.match(/ALTER\s+SEQUENCE\s+([\w\$]+)/i);
			//console.log(" ALTER+SEQUENCE:  ",msgg, 'as ',qrystr);
			if (msgg) {
				var gsv = getGenerator(zx, msgg[1], 0, 0);
				//console.log("SEQUENCE value :",gsv," ");
				if (gsv > 0) {
					qrystr = "";
					console.log("SEQUENCE", msgg[1], "already set");
				}
			}
			block.order = 600;
		} else if (name=qrystr.match(/CREATE\s+TABLE\s+([\w$]+)/i)) {
			block.name = 'TABLE_' + name[1];
			block.method = "TABLE";
			block.order = 700;
		} else if (name=qrystr.match(/CREATE_EXTEND\s+([\w$]+)\s+TABLE\s+([\w$]+)/i)) {
            qrystr = qrystr.replace(/CREATE_EXTEND\s+([\w$]+)/i, "CREATE");            
			block.name = 'TABLEEX_' + name[1] + "+" +name[2];
			block.method = "TABLE";
			block.order = 710;           
		} else if (name=qrystr.match(/CREATE\s+GLOBAL\s+TEMPORARY\s+TABLE\s+([\w$]+)/i)) {			
			block.name = 'TABLE_' + name[1];
			block.method = "TABLE";
			block.order = 800;
		} else if (name=qrystr.match(/CREATE\s+INDEX\s+([\w$]+)/)) {
			block.name = 'INDEX_' + name[1];
			block.expect = zx.dbu.sqltype(zx,/unsuccessful metadata update/,/ER_DUP_KEYNAME/,/failed because an index/);
			block.order = 900;
			//console.log("show CREATE INDEX:", qrystr);
		} else if (name=qrystr.match(/CREATE\s+VIEW\s+([\w$]+)/i)) {
			block.name = 'VIEW_' + name[1];
			qrystr = qrystr.replace(/CREATE\s+VIEW\s/i, "CREATE OR ALTER VIEW ");
			block.order = 1000;
		} else if (name=qrystr.match(/CREATE\s+OR\s+ALTER\s+VIEW\s+([\w$]+)/i)) { //execute as is
			block.name = 'VIEW_' + name[1];
			block.order = 1100;
		} else if (name=qrystr.match(/ALTER\s+VIEW\s+([\w$]+)/i)) { //execute as is
			block.name = 'VIEW_' + name[1];
			block.order = 1200;
		} else if (name=qrystr.match(/CREATE\s+EXCEPTION\s+([\w$]+)/i)) {
			block.name = 'EXCEPTION_' + name[1];
			qrystr = qrystr.replace(/CREATE\s+EXCEPTION\s/i, "CREATE OR ALTER EXCEPTION ");
			block.order = 1300;
		} else if (name=qrystr.match(/CREATE\s+OR\s+ALTER\s+EXCEPTION\s+([\w$]+)/i)) { //execute as is
			block.name = 'EXCEPTION_' + name[1];
			block.order = 1400;
		} else if (name=qrystr.match(/CREATE\s+(?:OR\s+ALTER\s+)?PROCEDURE\s+([\w$]+)/i)) {
			//console.log("\n\n\n\n========================================CREATE PROCEDURE");
			block.name = 'PROCEDURE_' + name[1];
			
			var DECLARE_PROCEDURE = deepcopy(block);
			if (zx.fb25)
			{
			    qrystr = qrystr.replace(/CREATE\s+PROCEDURE\s/i, "CREATE OR ALTER PROCEDURE ");
				DECLARE_PROCEDURE.method = "DECLARE_PROCEDURE";
				DECLARE_PROCEDURE.order = 850;
				DECLARE_PROCEDURE.qrystr = qrystr;
				DECLARE_PROCEDURE.Hash = zx.ShortHash(qrystr);
				blocks.push(DECLARE_PROCEDURE);				
			} else if (zx.mysql57) {
				qrystr = "\r\n"+qrystr+"\r\n ;\r\n";
				
				DECLARE_PROCEDURE.method = "DROP_PROCEDURE";
				DECLARE_PROCEDURE.order = 1499;
				DECLARE_PROCEDURE.qrystr = "DROP PROCEDURE IF EXISTS "+ name[1] +";";
				DECLARE_PROCEDURE.Hash = zx.ShortHash(qrystr);
				blocks.push(DECLARE_PROCEDURE);
			} else if (zx.mssql12) {
				qrystr = "\r\n"+qrystr+"\r\n ;\r\n";
				
				DECLARE_PROCEDURE.method = "DROP_PROCEDURE";
				DECLARE_PROCEDURE.order = 1499;
				DECLARE_PROCEDURE.qrystr = "DROP PROCEDURE IF EXISTS "+ name[1] +";";
				DECLARE_PROCEDURE.Hash = zx.ShortHash(qrystr);
				blocks.push(DECLARE_PROCEDURE);				
			} else throw new Error("dialect code missing");

			block.order = 1500;
		} else if (name=qrystr.match(/CREATE\s+(?:OR\s+ALTER\s+)?FUNCTION\s+([\w$]+)/i)) {
			//console.log("\n\n\n\n========================================CREATE FUNCTION");
			block.name = 'PROCEDURE_' + name[1];
			
			var DECLARE_FUNCTION = deepcopy(block);
			if (zx.fb25)
			{
			    qrystr = qrystr.replace(/CREATE\s+FUNCTION\s/i, "CREATE OR ALTER PROCEDURE ");
				DECLARE_FUNCTION.method = "DECLARE_FUNCTION_TODO"; //todo more 
				DECLARE_FUNCTION.order = 850;
				DECLARE_FUNCTION.qrystr = qrystr;
				DECLARE_FUNCTION.Hash = zx.ShortHash(qrystr);
				blocks.push(DECLARE_FUNCTION);				
			} else if (zx.mysql57) {
				qrystr = "\r\n"+qrystr+"\r\n ;\r\n";
				
				DECLARE_FUNCTION.method = "DROP_FUNCTION";
				DECLARE_FUNCTION.order = 1499;
				DECLARE_FUNCTION.qrystr = "DROP FUNCTION IF EXISTS "+ name[1] +";";
				DECLARE_FUNCTION.Hash = zx.ShortHash(qrystr);
				blocks.push(DECLARE_FUNCTION);
			}  else throw new Error("dialect code missing");

			block.order = 1500;
			
		} else if (name=qrystr.match(/ALTER\s+PROCEDURE\s+([\w$]+)/i)) {
			block.name = 'PROCEDURE_' + name[1];
			//execute as is
			block.order = 1700;
		//} else if (name=qrystr.match(/CREATE\s+TRIGGER\s+([\w$]+)/i)) {
			
		} else if (name=qrystr.match(/CREATE\s+TRIGGER\s+\`*\'*\"*([\w$]+)\'*\"*\`*/i)) {
			//console.log("\n\n\n\n========================================CREATE TRIGGER");
			block.name = 'TRIGGER_' + name[1];
			//var trigger_ = qrystr.match(/CREATE\s+TRIGGER\s(\S*)/i);			
			//console.log("\n\n\n\n====================TRIGGER_function :",name[1]);
			if (zx.fb25) { qrystr = qrystr.replace(/CREATE\s+TRIGGER\s/i, "CREATE OR ALTER TRIGGER ");
			} else if (zx.mysql57) {
				//qrystr = "LOCK TABLES t1 WRITE; DROP TRIGGER test.ins_sum;" + qrystr +"UNLOCK TABLES;";
				
				var Drop_trigger = deepcopy(block);
				Drop_trigger.method = "DROP TRIGGER";
				//Drop_trigger.order--;
				Drop_trigger.order = 860;
				Drop_trigger.qrystr = "DROP TRIGGER IF EXISTS "+name[1]+";";
				Drop_trigger.Hash = zx.ShortHash(qrystr);
				blocks.push(Drop_trigger);
				
				//qrystr = "DELIMITER ;;\n " + qrystr  +"\nDELIMITER ;\n";
		    } else throw new Error("dialect code missing");
			
			block.order = 1800;
		} else if (name=qrystr.match(/CREATE\s+OR\s+ALTER\s+TRIGGER\s+([\w$]+)/i)) { //execute as is
			block.name = 'TRIGGER_' + name[1];
			block.order = 1900;
		} else if (qrystr.match(/GRANT\s/)) {
			//block.name = 'DOMAIN_' + name[1];
			block.expect = /unsuccessful metadata update/;
			block.order = 2000;
		} else if (qrystr.match(/INSERT\s+MATCHING\s/i)) {
			//block.name = 'DOMAIN_' + name[1];
			block.order = 2100;
			var mi = qrystr.match(/INSERT\s+MATCHING\s+([\w\$,\s]+)/i);
			if (mi) {
				insertmatchingfield = mi[1];
				block.records = [];
				insertmatchingblock = block;
			}
			qrystr = "";
		} else if (name=qrystr.match(/INSERT\s+INTO\s+([\w\$,\s]+)/i)) {			
			block.name = 'INSERTINTO_' + zx.ShortHash(qrystr);
			block.order = 2200;
			qrystr = qrystr.replace(/;\s+$/g, '');
			if (insertmatchingfield !== '') {				

				if (zx.fb25) {
					qrystr = "update or " + qrystr + " matching(" + insertmatchingfield + ") ";
					block.qrystr = qrystr;				
					insertmatchingblock.records.push(block);					
				} else if (zx.mysql57||zx.mssql12) {	
					block.qrystr = qrystr;				
				
					var Drop_record = deepcopy(block);
					Drop_record.method = "Drop_record";
					Drop_record.order = 2100;
					Drop_record.qrystr = exports.insert_update_variation(zx, qrystr,insertmatchingfield);
					Drop_record.Hash = zx.ShortHash(qrystr);
					blocks.push(Drop_record);					
				} else throw new Error("dialect code missing");
	
			
			}

		} else if (qrystr.match(/UPDATE\s/i)) {
			//block.name = 'DOMAIN_' + name[1];
			block.order = 2300;
		} else if (qrystr.match(/SET\s+TERM\s/i)) {
			block.name = 'SETTERM';
			qrystr = "";
			block.order = 2300;			
		} else {
			block.name = 'UNKNOWNSQL_' + zx.ShortHash(qrystr);

            //console.log('blocks.forEach else:',qrystr.substring(0,50));
            var qrystrx = comment_suppress(qrystr).trim()
            block.order = 9999;
            if (qrystrx===''||qrystrx===';')
                {
                block.method = "bypass";
                }
                else
                {
			

			if (verbosity > 1) {
				console.log("\r\n\r\n Unexpected DDL, from line : ",
					block.src.LineNr, ' lines:', block.src.l + 1, 'file: ',block.src.src_obj.srcinfo.filename,
					'\r\ntext:', qrystr, 
					'\r\n after:',blocks[bi ? bi - 1 : 0].q);
					
				zx.error.log_SQL_warning(zx, "Unexpected DDL :" + qrystr, zx.line_obj);
				fs.writeFileSync(zx.output_folder + 'blocks.json', JSON.stringify(blocks,null,4));
				exports.commit(zx);				
				if (zx.exit_on_error) process.exit(2);
			}
            }
		}
        //console.log('Update block q2, :',block.order);
		block.qrystr = qrystr;
		block.Hash = zx.ShortHash(qrystr);
		//console.log('Update block, :',block.order);//, block.qrystr );
		LineNr += block.src.l;
		//exports.show_DDL(zx,"working",blocks);
	}

	//console.log('Prepare_DDL blocks.length :', blocks.length);
	//exports.show_DDL(zx,"Adding",blocks);
	exports.blocks = exports.blocks.concat(blocks);
	//process.exit(2);
	return blocks;
};

exports.show_DDL = function (zx, msg, blocks) {
	console.log("show_DDL------------------------------------>", msg, blocks.length);
	blocks.forEach(function (block, i) {
		//console.log(msg, i," : ");
		console.log(msg, i, block.order, block.src.BlockNr, block.method, block.Hash, zx.show_longstring(block.qrystr)); //qrystr||'');
		//if (block.qrystr===undefined)
		//        console.log('block.undefined:', i,block);
	});
	console.log("done show_DDL^^^^^^^^^^^^^^^^^^", msg);
}

exports.Sort_DDL = function (zx, blocks) {

	blocks.sort(function (a, b) {
		//console.log('Sort_DDL :', a.order , b.order);
		if (a.order !== b.order)
			return (a.order - b.order);
		else
			return (a.src.BlockNr - b.src.BlockNr);
	});
	var Hash = 0;
	var build_str = [];
	var build_exec_str = [];
	var block_hashes = {};
	
	//console.log('Sort_DDL blocks.length:', blocks.length);
	
	blocks.forEach(function (block, i) {
		var qrystr = block.qrystr;
        Hash += +block.Hash;
		if (block.name) {
			block_hashes[block.name] = block.Hash;		
			//console.log('block.Hashes :', block.name,exports.lastHashes[block.name],block_hashes[block.name]);
		}
        if (block.Hash===undefined) 
            console.log('block.Hash===undefined :', block);
        
		if (block.method === 'DECLARE_PROCEDURE')
			qrystr = "DECLARE" + zx.show_longstring(qrystr);
		build_str.push(" BORDER:" + block.order + " BNR:" + block.src.BlockNr + " method:" + block.method + " SRC:" + qrystr);
		
		build_exec_str.push("-- block "+ block.order + " BNR:" + block.src.BlockNr + " Name:" +block.name+" #"+block.Hash);
		build_exec_str.push(qrystr);
	});
	block_hashes["Complete"] = Hash;
	return {
		Hashes : block_hashes,
		build_str : build_str.join('\n'),
		build_exec_str : build_exec_str.join('\n\n'),
	};
};
var DECLARE_PROCEDURE = function (cx, qrystr) {

	var m = qrystr.match(/([\S\s]*?AS\s)/i);
	if (m === null)
		return;

	//qrystr = "\n--->>>\n" +m[1] + "\nBEGIN END" + "\nSET TERM ; ^\n---<<<\n\n";

	qrystr = m[1] + "\nBEGIN END" + "\n";

	exec_qry(cx, qrystr);

};

exports.Execute_DDL = function (zx, blocks) {
	//console.log('exports.Execute_DDL length:', blocks.length);
	var cx = {
		zx : zx
	};
	blocks.forEach(function (block, i) {
		if ((block.qrystr === undefined) || (block.qrystr === "")) {
			//console.log('block.qrystr===undefined:', i,block);
		} else if (exports.lastHashes[block.name]===block.Hash) {
			//console.log('block.Hashes match:', block.name,exports.lastHashes[block.name],block.Hash);
		} else {			
			var qrystr = block.qrystr;
			cx.expect = block.expect;
			//console.log('block.Hashes missmatch:', block.name,exports.lastHashes[block.name],block.Hash,'"'+qrystr+'"');
			//console.log('exports.Execute_DDL item:', i,block.method,(qrystr||'').substring(0,40),'q:',(block.q||'').substring(0,40));
			var ex = block.src.src_obj.srcinfo;
			//delete block.src.src_obj.body;
			//delete block.src.src_obj.nonkeyd;

			//delete ex.source;
			// delete ex.q.query;

			//console.log('exports.Execute_DDL item:', i,block.src.LineNr);
			//console.log('exports.Execute_DDL item:', i,ex);
			//process.exit(2);
			if (1) {
				zx.line_obj = block.src; //.src_obj;
			} else {
				zx.line_obj.srcinfo = {};
				//in case the parser needs to throw an error
				zx.line_obj.srcinfo.main_page_name = ex.main_page_name;
				zx.line_obj.srcinfo.file_stack = ex.file_stack.slice(0);
				zx.line_obj.srcinfo.filename = ex.filename;
				//zx.line_obj.srcinfo.source = sourcestr;
				zx.line_obj.srcinfo.start_line = ex.start_line + block.src.LineNr;
				zx.line_obj.srcinfo.start_col = ex.start_col;
				zx.line_obj.srcinfo.current_tag_index = ex.current_tag_index;
				zx.line_obj.tag = block.src.src_obj.tag;
			}
			if (block.method === "TABLE") {
				
                CREATE_TABLE(zx,qrystr);
			} else if (block.method === "DECLARE_PROCEDURE") {
				qrystr = DECLARE_PROCEDURE(cx, qrystr);
			} else if (block.method === "bypass") {}
			else
				if (qrystr !== "") {
					//console.log(" Execute DDL from line ", block.src.LineNr, ' for ', block.src.l + 1, ' lines '); //'as ',qrystr);
					//			console.log(" Execute DDL from line ", LineNr, ' for ', block.src.l + 1, ' lines as ',qrystr);
					exec_qry(cx, qrystr);
				} else {
					//console.log(" Skipped DDL from line ", block.src.LineNr, ' for ', block.src.l + 1, ' lines '); //'as ',qrystr);
				}
		}
	});

};

function full_updgade(zx, ddl_filename_prefix) {
	/*
	global con
	try:

	con = fdb.connect(
	host=zxconf.get('zxconf', 'SQLServer', 0), database=(zxconf.get('zxconf', 'DatabaseFolder', 0)+zxconf.get('zxconf', 'DatabaseFile', 0)),
	user=zxconf.get('zxconf', 'UserName', 0), password=zxconf.get('zxconf', 'Password', 0),
	sql_dialect=3

	)
	#con.execute_immediate("SET SQL DIALECT 3")
	print 'dialect:',con.sql_dialect,'  Server version:',con.version,' FDB version',fdb.__version__

	except fdb.fbcore.DatabaseError:
	errors=str(sys.exc_info()[1])
	if (errors.find('335544344')>=0 and errors.find('No such file or directory')>=0):
	print "No such file or directory - attempting to create database";
	con = fdb.create_database(
	host=zxconf.get('zxconf', 'SQLServer', 0), database=(zxconf.get('zxconf', 'DatabaseFolder', 0)+zxconf.get('zxconf', 'DatabaseFile', 0)),
	user=zxconf.get('zxconf', 'UserName', 0), password=zxconf.get('zxconf', 'Password', 0),
	page_size=8192
	)
	else:
	raise sys.exc_info()[1], None, sys.exc_info()[2] #re throw
	 */

	/*    if os.path.exists('make.sql'):
	#only break if we make - else we just load
	dropTriggers()
	Execute_DDL('clean');
	con.commit()
	Execute_DDL('make');
	con.commit()

	Execute_DDL('load');
	// con.commit()
	Execute_DDL('load-b');
	Execute_DDL('load-c');
	Execute_DDL('load-d');
	 */

	//con.commit()
}

exports.Backup_DDL = function (zx, reflect, backup) {
    if (!zx.config.db.schema_backup) return;
	var result = zx.dbu.extract_dll(zx);
	//console.log('extract_dll result is ',str.ddl);
	//console.trace('backup extract_dll result is ', result.err);
	if (backup) {
		var d = new Date();
		var df = (1900 + d.getYear()) + '_' + d.getMonth() + '_' + d.getDate() + ' ' + d.getHours() + '_' + d.getMinutes() + '_' + d.getSeconds();
		fs.writeFileSync(zx.output_folder + '/Audit/Before update on ' + df, result.ddl);
	}
	if (reflect) {
		fs.writeFileSync(zx.output_folder + 'reflect.sql', result.ddl);
	}

}

exports.update = function (zx) {
	//called several times from the input processor....exports.Prepare_DDL(zx, null, model_text)
	//console.log('exports.blocks length:', exports.blocks.length);

	if (exports.lastHash === null)
		try {
			exports.lastHashes = JSON.parse(require('fs').readFileSync(zx.output_folder + 'update.hash').toString());
			exports.lastHash = 'x'+exports.lastHashes.Complete;
		} catch (e) {}

	//exports.show_DDL(zx,"B4 Sort",exports.blocks);

	exports.rebuild = 1;
	var B = exports.Sort_DDL(zx, exports.blocks);

	fs.writeFileSync(zx.output_folder + 'prebuild.sql', B.build_str);
	fs.writeFileSync(zx.output_folder + 'prebuild_exec.sql', B.build_exec_str);
	//console.log('exports.Sort_DDL length:', exports.blocks.length,exports.lastHash,Hash);
	//exports.show_DDL(zx,"Afer Write",exports.blocks);
    
    
    if ((exports.lastHash !== null) && (exports.lastHash === B.Hashes["Complete"]))
       {
       exports.write_log.push("Model meta hash indicates it has not changed..."+ B.Hashes["Complete"]);
       //console.log("Model meta hash indicates it has not changed...", B.Hash);
       }
    
	if ((exports.lastHash === null) || (exports.lastHash !== B.Hashes["Complete"]) || (zx.config.db.schema_rebuild==="always")) {
		//console.log("Model meta hash indicates it has changed...");
	    //console.log('exports.Execute_DDL hashed:',exports.lastHash,"\n   B.Hash 210425:",B.Hash);
		//console.log('Backup_DDL');
		exports.Backup_DDL(zx, 0, 1); //audit trail
		//console.log('Backup_DDL-done');
	    //console.log('exports.Execute_DDL hashed aaa:');	
		//exports.show_DDL(zx, "After sort b4 exec ", exports.blocks);
	    //console.log('exports.Execute_DDL hashed bbb:');	
        //console.log('Update DDL :')        
		exports.Execute_DDL(zx, exports.blocks, 0);
        //console.log('Update DDL done :');		
        //console.log('exports.Execute_DDL hashed ccc:');			
		exports.Backup_DDL(zx, 1, 0); //reflection.sql
        //console.log('exports.Execute_DDL hashed ddd:');	
        //process.exit(2);		
	}
	//console.log('exports.write_log.length  :', exports.write_log.length);
	exports.lastHash = B.Hashes["Complete"];
	exports.lastHashes = extend(exports.lastHashes, B.Hashes); //second one has the priority
	fs.writeFileSync(zx.output_folder + 'update.hash', JSON.stringify(B.Hashes,null,4));
	exports.commit(zx);
}
  
exports.commit = function (zx) {  
	fs.writeFileSync(zx.output_folder + 'input.sql', exports.input_audit.join('\r\n'));
	fs.writeFileSync(zx.output_folder + 'build.sql', exports.write_log.join('\r\n'));
	
};

function recreate(zx, model_text) {}

exports.init = function (zx) {

	exports.lastHash = null;
	exports.lastHashes = {};
	exports.src_obj = {};
	exports.blocks = [];
	exports.write_log = [];
	exports.input_audit = [];

};

exports.start_up = function (zx) {};

exports.unit_test = function (zx) {

	console.log('comment_suppress', comment_suppress("abc /* cdef \n  hij */  klm \n nmp/* and */ stuff"));

	console.log('checkTable(" Z$USER "):', checkTable(zx, " Z$USER "));
	var cx = {
		zx : zx,
		expect : /335544569/
	};
	exec_qry(cx, " ALTER TABLE " + " xTable " + " ADD " + " field varchar(20); ");
	cx = {
		zx : zx,
		expect : /335544561/
	};
	exec_qry(cx, " ALTER TABLE " + " xTable " + " ADD " + " field varchar(20); ");

	console.log('getGenerator:', getGenerator(zx, 'Z$CONTEXT_SEQ', 0));

	//console.log('CREATE_TABLE:', exports.CREATE_TABLE(zx,
	//		" CREATE TABLE TODO_MVC(REF PK not null, OWNER fk, NAME VARCHAR(100), STATUS VARCHAR(10), CREATED_STAMP TIMESTAMP DEFAULT 'now'); "));

	console.log('getGenerator:', getGenerator(zx, 'Z$CONTEXT_SEQ', 0));

	exports.Execute_DDL(zx, " / home / xie01 / Sites / sql / sql - mvc / install / demo_db_dll_x.sql ");
	//dropTriggers(zx);

	var result = zx.dbu.extract_dll(zx);
	//console.log('extract_dll result is ',str.ddl);
	console.log('unit test extract_dll result is ', result.err);

	//process.exit(2);

	//exports.Execute_DDL(zx, "/home/xie01/Sites/sql/sql-mvc/install/demo_db_dll_x.sql");

	//console.log('CREATE_TABLE:', exports.CREATE_TABLE(zx,
	//		" CREATE TABLE TODO_MVC(REF PK not null, OWNER fk, NAME VARCHAR(100), STATUS VARCHAR(10), CREATED_STAMP TIMESTAMP DEFAULT 'now'); "));
	//dropTriggers(zx);


	//var res = zx.dbu.singleton.future(null, cx, " query ",
	//" select count( * )from rdb$relations ",0);
	//console.log('Query result is ',res.result[0].count);

	//process.exit(2);


};

