Module:Lang/data/make is latn data

require ('strict');

local title_object = mw.title.getCurrentTitle ();								-- get this module's title object
if not title_object.fullText:find ('/doc$') then								-- are we are looking at the ~/doc page or the module page?
	local module_doc_title = title_object.fullText .. '/doc';					-- looking at the module page so make a page name for this module's doc page
	title_object = mw.title.new (module_doc_title);								-- reset title object to this module's doc page
end
local content = title_object:getContent();										-- get the doc page content

local common_scripts_singles_t = {};											-- these used when constructing final output
local common_scripts_ranges_t = {};
local latn_scripts_singles_t = {};
local latn_scripts_ranges_t = {};
local extension_scripts_singles_t = {};
local extension_scripts_ranges_t = {};


--[[--------------------------< Z Y Y Y _ L A T N _ C O D E P O I N T S _ G E T >-----------------------------

extract zyyy-script (common) and latn codepoints from Module:Unicode data/scripts.  There are individual codepoints
and ranges of codepoints.

]]

local function zyyy_latn_codepoints_get ()
	local unicode_scripts = mw.loadData ('Module:Unicode data/scripts');

	for code_point, script in pairs (unicode_scripts.singles) do				-- spin through the ~/scripts.singles table
		if 'Latn' == script then
			latn_scripts_singles_t[code_point] = true;							-- not a sequence so we can check for duplicates later
		elseif 'Zyyy' == script then
			common_scripts_singles_t[code_point] = true;						-- not a sequence so we can check for duplicates later
		end
	end
	
	for i, code_points_t in ipairs (unicode_scripts.ranges) do					-- spin through the ~/scripts.ranges table
		if 'Latn' == code_points_t[3] then
			table.insert (latn_scripts_ranges_t, {code_points_t[1], code_points_t[2]});
		elseif 'Zyyy' == code_points_t[3] then
			table.insert (common_scripts_ranges_t, {code_points_t[1], code_points_t[2]});
		end
	end
end


--[[--------------------------< E X T E N S I O N _ C O D E P O I N T S _ G E T >-----------------------------

read a local copy of the current unicode ScriptExtensions-xx.x.x.txt file (hidden in this module's doc page).
extract latn-script codepoints and ranges.  Convert codepoints from hex to decimal (same format as codepoints
extracted from Unicode data/scripts).

]]

local function extension_codepoints_get ()
	local line_pattern = '%x+[^\r\n]+';

	for line in content:gmatch (line_pattern) do								-- read each line of extensions text file
		local single = line:match ('(%x+)%s*;[^#]*Latn[^#]*#%s*%a%a%s*(.+)');
		if single then
			extension_scripts_singles_t[tonumber ('0x' .. single)] = true;		-- convert hex index to decimal and save
		end

		local range_t = {};														-- a single codepoint-range
		range_t[1], range_t[2] = line:match ('(%x+)%.%.(%x+)%s*;[^#]*Latn[^#]*#%s*%a%a%s*%[%d+%]%s*(.+)');
		if range_t[1] then
			range_t[1] = tonumber ('0x' .. range_t[1]);							-- convert hex index to decimal
			range_t[2] = tonumber ('0x' .. range_t[2]);							-- convert hex index to decimal
			table.insert (extension_scripts_ranges_t, range_t);					-- and save
		end
	end
end


--[[--------------------------< B I N A R Y _ S E A R C H >---------------------------------------------------
]]

local function binary_search (target, ranges_t)
	local idx_bot = 1;															-- initialize to index of first key
	local idx_top = #ranges_t;													-- initialize to index of last key (number of keys)

	if (target < ranges_t[idx_bot][1]) or (target > ranges_t[idx_top][2]) then	-- invalid; target out of range
		return;																	-- TODO: return something meaningful?
	end

	local idx_mid;
	local flag = false;

	while 1 do
		idx_mid = math.ceil ((idx_bot + idx_top) / 2);							-- get the mid-point in the sequence
		if (target >= ranges_t[idx_mid][1]) and (target <= ranges_t[idx_mid][2]) then		-- indexed low value <= target <= indexed high value
			return true;														-- we found the range that holds the <target> character; return true
		
		elseif (target > ranges_t[idx_mid][2]) then								-- is <target> > indexed high value?
			idx_bot = idx_mid;													-- adjust <idx_bot> up

		else																	-- here when <target> less than indexed low value
			idx_top = idx_mid - 1;												-- adjust <idx_top> down
		end

		if flag then
			break;																-- here when we just evaluated the last range and <target> not found
		end
	
		if not flag and (idx_bot == idx_top) then								-- set true just before we evaluate the last range
			flag = true;
		end
	end
end


--[[--------------------------< E X P A N D _ R A N G E >-----------------------------------------------------

expand range <range_t>[1] to <range_t>[2] into <out_t> as singles:
	{10, 15} -> {10, 11, 12, 13, 14, 15}
	
]]

local function expand_range (range_t, out_t)
	for i=range_t[1], range_t[2] do
		table.insert (out_t, i);
	end
end


--[[--------------------------< M A K E _ R A N G E S _ F R O M _ S I N G L E S >------------------------------

search <scripts_singles_t> and for ranges of contiguous codepoints to be added to the ranges list.  Singles
ranges added to the ranges will be removed from the final singles list later during output formatting.

]]

local function make_ranges_from_singles (scripts_singles_t, ranges_from_singles_t)
	local singles_t = {};														-- sequence of singles suitable for sorting
	for k, _ in pairs (scripts_singles_t) do
		table.insert (singles_t, k);											-- add codepoint to singles_t
	end
	
	table.sort (singles_t);														-- ascending sort to get them all in increasing order
	local bottom, top;															-- bottom and top of extracted range

	for i, single in ipairs (singles_t) do
		if not bottom and (single + 1) == singles_t[i+1] then					-- if (singles_t[i]+1) same value as next element ([i+1])
			bottom = single;													-- set new range bottom value
			top = singles_t[i+1];												-- set new range top value
			table.remove (singles_t, i);										-- remove range bottom value from table (<i> now indexes top range value)

			while (top + 1) == singles_t[i+1] do								-- if (singles_t[i]+1) same value as next element ([i+1])
				top = singles_t[i+1];											-- set new top
				table.remove (singles_t, i);									-- remove range bottom value from table (<i> now indexes new top range value)
			end
		end
		
		if bottom then															-- not nil when we have extracted a range
			mw.log (string.format ('%s–%s (%.4X..%.4X) extracted from singles_t', bottom, top, bottom, top));
			table.insert (ranges_from_singles_t, {bottom, top})					-- save the extracted range
			bottom = nil;														-- unset these for the next range
			top = nil;
		end
	end
end


--[[--------------------------< M A I N >---------------------------------------------------------------------

{{#invoke:Sandbox/trappist the monk/is latn|main}}

build composite lists (single and ranges) of common- and latn-script codepoints.

Duplicates are singles and ranges swallowed.

When a range has a different length from another range with the same starting point, this function takes the
longest range.

When a range is a subset of a larger range, the subset range is removed from the list.

Contiguous ranges (ending codepoint of one range is one less than the starting codepoint of the next range) are
joined to make a single range.

Expands all ranges into singles and combines with separately defined singles to create one long list of singles
because why not?

Finally the lists are made all pretty-like and rendered for copy pasta into an appropriate data module for use
by Module:Lang.

TODO: detect and remove overlapping ranges where one range starts in one range and ends in another range?
TODO: there are contiguous codepoints listed in the singles list; combine these into ranges

]]

local function main (frame)
	zyyy_latn_codepoints_get();													-- get common- and latn-script codepoints from [[Module:Unicode data/scripts]]
	extension_codepoints_get();													-- get latn-script codepoints from local copy of unicode scripts text file
	
	local scripts_singles_t = {};
	for _, scripts_t in ipairs ({latn_scripts_singles_t, common_scripts_singles_t, extension_scripts_singles_t}) do
		for k, v in pairs (scripts_t) do
			scripts_singles_t[k] = v;											-- duplicates (if any) are swallowed
		end
	end

	local ranges_from_singles_t = {};											-- a sequence of sequences
	make_ranges_from_singles (scripts_singles_t, ranges_from_singles_t);		-- add contiguous singles in <scripts_singles_t> to <ranges_from_singles_t>

	local temp_t = {};															-- for ranges; <k> is range low value, <v> is range high value
	for _, ranges_t in ipairs ({latn_scripts_ranges_t, common_scripts_ranges_t, extension_scripts_ranges_t, ranges_from_singles_t}) do
		for _, range_t in pairs (ranges_t) do
			if temp_t[range_t[1]] then
				if temp_t[range_t[1]] ~= range_t[2] then
					mw.log (range_t[1] .. '–' .. range_t[2] .. string.format (' (%x..%x) ', range_t[1], range_t[2]) .. 'does not match: ' .. temp_t[range_t[1]] .. string.format (' (%x)', temp_t[range_t[1]]));
					if temp_t[range_t[1]] > range_t[2] then
						range_t[2] = temp_t[range_t[1]];						-- use the greater high value
					end
				else
					mw.log (range_t[1] .. '–' .. range_t[2] .. string.format (' (%x..%x) ', range_t[1], range_t[2]) .. ' is duplicate');	-- log and overwrite existing range
				end
			end
			temp_t[range_t[1]] = range_t[2];									-- add to temp table
		end
	end

	local scripts_ranges_t = {};
	for k, v in pairs (temp_t) do												-- make a sequence of codepoint range sequences
		table.insert (scripts_ranges_t, {k, v});
	end
	local function sort (a_t, b_t)												-- local function to ascending sort range tables
		return a_t[1] < b_t[1];
	end
	
	table.sort (scripts_ranges_t, sort);										-- ascending sort the range sequence

	for k, v_t in ipairs (scripts_ranges_t) do									-- remove ranges that are subsets of other ranges; must be sorted first
		if k == #scripts_ranges_t then
			break;																-- done because there is no scripts_ranges_t[k+1]
		end
		
		if (scripts_ranges_t[k+1][1] < v_t[2]) and (scripts_ranges_t[k+1][2] < v_t[2]) then	-- next range start and end less than current range end
			mw.log ('removed subrange' .. scripts_ranges_t[k+1][1] .. '–' .. scripts_ranges_t[k+1][2] .. string.format (' (%x..%x) ', scripts_ranges_t[k+1][1], scripts_ranges_t[k+1][2]));
			table.remove (scripts_ranges_t, k+1)
		end
	end
	
	local i = 1;																-- indexer
	while i ~= #scripts_ranges_t do												-- join contiguous ranges into a single range;
		if (scripts_ranges_t[i][2] + 1) == scripts_ranges_t[i+1][1] then		-- example: if {0, 64+1} == {65, 90} then join
			mw.log (string.format ('joined: %s..%s and %s..%s', scripts_ranges_t[i][1], scripts_ranges_t[i][2], scripts_ranges_t[i+1][1], scripts_ranges_t[i+1][2]))
			scripts_ranges_t[i][2] = scripts_ranges_t[i+1][2];					-- join
			table.remove (scripts_ranges_t, i+1);								-- remove joined
		else
			i = i+1;															-- not contiguous, bump the indexer
		end
	end

	local singles_out_t = {};													-- sequence to hold singles_out_t
	local expanded_out_t = {};													-- sequence to hold singles + explanded ranges
	for k, _ in pairs (scripts_singles_t) do
		if binary_search (k, scripts_ranges_t) then								-- omit singles that are included in ranges_out_t
			mw.log (string.format ('removed: %s (%X)', k, k));
		else
			table.insert (singles_out_t, k);
			table.insert (expanded_out_t, k);
		end
	end		
	table.sort (singles_out_t);
	for i, v in ipairs (singles_out_t) do
		local single_str = string.format ('[%s] = true,', v);
		local rep = math.ceil ((80 - (4 + single_str:len())) / 4);
		singles_out_t[i] = string.format ('\t%s%s-- %.4X', single_str, string.rep ('\t', rep), v);
	end		
	table.insert (singles_out_t, 1, '<syntaxhighlight lang="lua">local singles_t = {');	-- opening stuff
	table.insert (singles_out_t, '\t}</syntaxhighlight>');						-- to close the table

	local ranges_out_t = {};
	for _, v_t in ipairs (scripts_ranges_t) do
		local range_str = string.format ('{%s, %s},', v_t[1], v_t[2]);
		local rep = math.ceil ((80 - (4 + range_str:len())) / 4);
		table.insert (ranges_out_t, string.format ('\t%s%s-- %.4X..%.4X', range_str, string.rep ('\t', rep), v_t[1], v_t[2]));
		expand_range (v_t, expanded_out_t);										-- expand this range into <expanded_out_t>
	end		
	table.insert (ranges_out_t, 1, '<syntaxhighlight lang="lua">local ranges_t = {');	-- opening stuff
	table.insert (ranges_out_t, '\t}</syntaxhighlight>');						-- to close the table

	table.sort (expanded_out_t);
	for i, v in ipairs (expanded_out_t) do
		local single_str = string.format ('[%s] = true,', v);
		local rep = math.ceil ((80 - (4 + single_str:len())) / 4);
		expanded_out_t[i] = string.format ('\t%s%s-- %.4X', single_str, string.rep ('\t', rep), v);
	end		
	table.insert (expanded_out_t, 1, '<syntaxhighlight lang="lua">local singles_t = {');	-- opening stuff
	table.insert (expanded_out_t, '\t}</syntaxhighlight>');						-- to close the table
	
	return frame:preprocess (table.concat ({									-- make a big string and done
		table.concat (singles_out_t, '\n'),
		'\n\n',
		table.concat (ranges_out_t, '\n'),
		'\n\n',
		table.concat (expanded_out_t, '\n'),
		}));
end


--[[--------------------------< E X P O R T S >---------------------------------------------------------------
]]

return {
	main = main,
	}