မော်ဂျူး:ja-parse

Documentation for this module may be created at မော်ဂျူး:ja-parse/doc

local export = {}

local len = mw.ustring.len
local sub = mw.ustring.sub
local gsub = mw.ustring.gsub
local find = mw.ustring.find
local match = mw.ustring.match
local gmatch = mw.ustring.gmatch

local m_ja = require('Module:ja')

local kanji_pattern = "㐀-䶵一-鿌\239\164\128-\239\171\153𠀀-𯨟"
local kana_pattern = 'ぁ-ゖァ-ヺー'
local japanese_pattern = kana_pattern .. kanji_pattern .. 'ａ-ｚＡ-Ｚ０-９〆々'

local headword_templates = {
	['ja-adj'] = true, ['ja-pos'] = true, ['ja-noun'] = true, ['ja-phrase'] = true,
	['ja-verb'] = true, ['ja-verb form'] = true, ['ja-verb-suru'] = true, 
}

local function find_headword_template(wikitext)
	local index =
		wikitext:find('{{ja%-adj[|}]') or
		wikitext:find('{{ja%-pos[|}]') or
		wikitext:find('{{ja%-noun[|}]') or
		wikitext:find('{{ja%-phrase[|}]') or
		wikitext:find('{{ja%-verb[|}]') or
		wikitext:find('{{ja%-verb form[|}]') or
		wikitext:find('{{ja%-verb%-suru[|}]')
	if index then
		-- This assumes that the template has matching braces.
		return wikitext:match('%b{}', index)
	end
end

local function parse_template(wikitext) -- only supports the simplest format
	local template = wikitext
	template = template:gsub('%[%[([^%[%]|]-)|([^%[%]|]-)%]%]', '[[%1`%2]]')
	local name
	local args = {}
	for glob in mw.text.gsplit(template:gsub('^{{', ''):gsub('}}$', ''), '|') do
		if not name then
			name = glob
		else
			glob = glob:gsub('`', '|')
			local key, value = match(glob, "(.-)=(.*)")
			if key and value then
				args[key] = value
			else
				table.insert(args, glob)
			end
		end
	end
	return name, args
end

local function contains(list, item)
	for i = 1, #list do
		if list[i] == item then return true end
	end
	return false
end


-- A function to parse Japanese entries, returning a list of etym sections, each having the form { wikitext, type = ( 'lemma' | 'redirect' | '' ), spellings = <a list of the term's modern spellings>, historical_spellings = <a list of the term's historical kana spellings> }. In case of multiple etymologies, each ===Etymology n=== part constitutes an etym section. Otherwise, the whole Japanese section minus any ===Kanji [n]=== subsections constitutes a single etym section.
-- Note: The function divides sections strictly by L3 headers. As a result:
-- (1) If an entry describes both a kanji and a single word, any templates beginning the word (such as {{ja-spellings}}) will be erroneously considered part of the kanji section above. This function only remedies the cases of {{ja-spellings}} and {{ja-kanjitab}}, by inserting an empty header === === above it before parsing. (This problem is absent for entries with multiple etymologies, since each word must begin with ===Etymology n===.)
-- (2) If an entry describes multiple words, word-specific templates such as {{topics|ja|Biology}} must now be placed at the end of the relevant word instead of the whole entry. If they are put at the end of the ==Japanese== entry, they will be either erroneously considered part of the final word or additional sections such as ===References===, and ignored when {{ja-see}} copies categories around.

function export.extract_etym_sections(lemma)
	local page = mw.title.new(lemma):getContent() or ''
	--local l2 = mw.ustring.match(page .. '\n----\n', '==ဂျပန်==\n(.-)%-%-%-%-')
	local l2 = mw.ustring.match(page, '==ဂျပန်==\n(.-)%-%-%-%-') or mw.ustring.match(page, '==ဂျပန်==\n(.*)')
	
	-- split into L3 sections
	local l3_sections = {}
	local multi_etym = false
	
	if l2 then
		-- special hack mentioned above
		if not find(l2, '===ရင်းမြစ် ၁===') and (find(l2, '===ခန်ဂျိ===') or find(l2, '===ခန်ဂျိ %d+===')) then
			l2 = gsub(l2, '{{ja%-spellings', '=== ===\n{{ja-spellings')
			l2 = gsub(l2, '{{ja%-kanjitab', '=== ===\n{{ja-kanjitab')
		end
		
		local current_l3_title = ''
		local current_l3_content = {}
		for v in l2:gmatch('[^\n]+') do
			if find(v, '^===[^=]') then
				table.insert(l3_sections, { current_l3_title, table.concat(current_l3_content, '\n') })
				current_l3_title = match(v, '^===([^=]+)')
				if current_l3_title == 'ရင်းမြစ် ၁' then multi_etym = true end
				current_l3_content = {}
			end
			table.insert(current_l3_content, v)
		end
		table.insert(l3_sections, { current_l3_title, table.concat(current_l3_content, '\n') })
	end
	
	-- group the L3 sections into etym sections
	local etym_sections = {}
	if multi_etym then
		for _, v in ipairs(l3_sections) do
			local header = v[1]
			local content = v[2]
			if find(header, '^ရင်းမြစ် %d+$') then
				table.insert(etym_sections, content)
			end
		end
	else
		local word = {}
		for _, v in ipairs(l3_sections) do
			local header = v[1]
			local content = v[2]
			if not (header == 'ခန်ဂျိ' or find(header, '^ခန်ဂျိ %d+$')) then
				table.insert(word, content)
			end
		end
		word = table.concat(word, '\n')
		table.insert(etym_sections, word)
	end
	
	-- finally, determine the type of each etym section
	for i = 1, #etym_sections do
		etym_section = etym_sections[i]
		local ja_see = find(etym_section, '{{ja%-see[|}]') or find(etym_section, '{{ja%-see-kango[|}]')
		if ja_see then
			local spellings = { lemma }
			for v in gmatch(match(etym_section, '.-}}', ja_see), '[' .. japanese_pattern .. ']+') do
				table.insert(spellings, v)
			end
			etym_sections[i] = { etym_section, type = 'redirect', spellings = spellings, historical_spellings = {} }
		else
			local ja_forms = etym_section:find('{{ja%-spellings[|}]')
			if ja_forms then
				local spellings = { lemma }
				local historical_spellings = {}
				local name, args = parse_template(etym_section:match('%b{}', ja_forms))
				for i = 1, #args do 
					table.insert(spellings, args[i])
				end
				table.insert(historical_spellings, args['h'])
				for i = 2, 5 do 
					table.insert(historical_spellings, args['h' .. i])
				end
				if args['h6'] then error('ja-parse: I don\'t support more than five historical spellings at the moment. Please expand me.') end
				etym_sections[i] = { etym_section, type = 'lemma', spellings = spellings, historical_spellings = historical_spellings }
			else
				local headword_template = find_headword_template(etym_section)
				if headword_template then
					local spellings = { lemma }
					local historical_spellings = {}
					local name, args = parse_template(headword_template)
					for i = 1, #args do
						local candidate = args[i]
						if find(candidate, '[' .. japanese_pattern .. ']') then
							table.insert(spellings, m_ja.remove_ruby_markup(candidate))
						end
					end
					table.insert(historical_spellings, args.hhira)
					table.insert(historical_spellings, args.hkata)
					etym_sections[i] = { etym_section, type = 'lemma', spellings = spellings, historical_spellings = historical_spellings }
				else
					etym_sections[i] = { etym_section, type = '', spellings = {}, historical_spellings = {} }
				end
			end
		end
	end
	return etym_sections
end

-- A function to parse Japanese entries based to the function above, but filters the result and finds the etym section with the alternative spelling given by the spelling, and returns it as wikitext.
function export.get_etym_section(lemma, spelling)
	local words = export.extract_etym_sections(lemma)
	local result = {}
	local spellings = {}
	for _, v in ipairs(words) do
		if v.type == 'lemma' and contains(v.spellings, spelling) then
			table.insert(result, v[1])
			for _, spelling in ipairs(v.spellings) do
				if not contains(spellings, spelling) then table.insert(spellings, spelling) end
			end
		end
	end
	local wikitext = table.concat(result, '\n')
	return wikitext, spellings
end

function export.extract_definitions_and_categories(wikitext, lemma, alt_spelling, frame)
	local def = {}
	local cat = {}
	local current_section = ''
	
	for v in wikitext:gmatch('[^\n]+') do
		if v:find('^#+ ') then
			if not v:find('{{rfdef') and not (v:find('{{ja%-def|') and not v:find('|' .. alt_spelling .. '[|}]') and find(alt_spelling, '[' .. kanji_pattern .. ']')) then
				table.insert(def, { v, pos = current_section })
			end
		elseif v:find('^===') then
			current_section = v:gsub("^=*(.-)=*$", "%1")
		else
			table.insert(cat, v)
		end
	end
	
	-- expand the other parts for categories
	local cat = table.concat(cat, '\n')
	cat = gsub(cat, '<ref', '')
	local function process_template_header(a, b) -- if the template begins with "{{ja-usex|", a is "ja-usex" and b is "|". 
		local templates_to_exclude = {
			-- These templates are ignored as an optimization since they don't generate categories.
			['m'] = true, ['l'] = true, ['ja-l'] = true, ['ja-r'] = true, ['gloss'] = true,
			['w'] = true, ['wp'] = true, ['swp'] = true, ['wikipedia'] = true,
			['lang'] = true, ['furigana'] = true, ['wj'] = true, ['lj'] = true, ['ruby/ja-w2'] = true, ['ruby/ja'] = true, ['ruby'] = true,
			['ja-kanji forms'] = true, ['w2'] = true, ['sense'] = true, 
			['IPAfont'] = true, ['IPAchar'] = true,
			['ja-adj-infl'] = true, ['ja-i'] = true, ['ja-na'] = true, ['ja-adjdecl'] = true, ['ja-decl-na'] = true, ['ja-go-bu'] = true, ['ja-go-gu'] = true, ['ja-go-ku'] = true, ['ja-go-mu'] = true, ['ja-go-nu'] = true, ['ja-go-ou'] = true, ['ja-go-ru'] = true, ['ja-go-su'] = true, ['ja-go-tsu'] = true, ['ja-go-u'] = true, ['ja-honorific'] = true, ['ja-ichi'] = true, ['ja-kuru'] = true, ['ja-suru'] = true, ['ja-suru-i-ku'] = true, ['ja-suru-tsu'] = true, ['ja-verbconj'] = true, ['ja-verbconj-auto'] = true, ['ja-verbconj-row'] = true, ['ja-verbconjugation'] = true, ['ja-zuru'] = true, 
			['ja-kanji spellings'] = true, ['ja-ks'] = true, ['ja-spellings'] = true, ['ja-forms'] = true,
			['Japanese first-person pronouns'] = true, ['der-top'] = true, ['der-bottom'] = true, 
			['der-mid'] = true, ['der-top3'] = true, ['der-top4'] = true, ['der-top5'] = true, ['rel-top'] = true, 
			['ja-uk'] = true,
			
			-- These templates are ignored since they generate categories that are spelling-specific or that we're not interested in
			['ja-kanjitab'] = true, ['ateji'] = true, ['ja-ateji'] = true, ['ja-kanji'] = true, ['ja-readings'] = true, 
			['juku'] = true, ['jukujikun'] = true, ['ja-jukujikun'] = true, 
			['ja-def'] = true, ['synonyms'] = true,
		}
		if templates_to_exclude[a] then
			return '{{=' .. b
		elseif headword_templates[a] then
			local source_script = m_ja.script(lemma)
			if source_script == 'Hira' or source_script == 'Kana' or source_script == 'Hira+Kana' then
				return '{{' .. a .. '|hira=' .. lemma .. b
			else
				return '{{' .. a .. b
			end
		elseif a == 'ja-usex' or a:find('^quote') then -- special hack
			return '[[Category:နမူနာအသုံး ပါဝင်သော ဂျပန် ဝေါဟာရများ]]{{=' .. b
		else
			return '{{' .. a .. b
		end
	end
	cat = gsub(cat, '{{([^|}\n]+)\n?([|}])', process_template_header)
	cat = gsub(cat, '{{ja%-pron.-}}', function(pron)
		local result = ''
		if not find(pron, '|noipa=') then result = result .. '[[Category:IPA အသံထွက် ပါဝင်သော ဂျပန် ဝေါဟာရများ]]' end
		if find(pron, '|a=') or find(pron, '|audio=') then result = result .. '[[Category:အသံလင့်ခ်များ ပါဝင်သော ဂျပန် ဝေါဟာရများ]]' end
		return result
		end)
	cat = frame:preprocess(cat)
	
	local cat2 = {}
	for i in gmatch(cat, '%[%[Category:.-%]%]') do table.insert(cat2, i) end
	cat = table.concat(cat2)
	-- one might want to modify the sortkeys here
	
	return def, cat
end

--[[
function export.fetch_reading(title, _wikitext) -- the _wikitext parameter is only used for recursion
	local wikitext
	if _wikitext then
		wikitext = _wikitext
	else
		local page = mw.title.new(title):getContent() or ''
		local l2 = page:match('==ဂျပန်==\n(.-)%-%-%-%-') or page:match('==ဂျပန်==\n(.*)') or ''
		if l2 == '' or l2:find('===ရင်းမြစ် ၁===') then
			wikitext = ''
		else
			wikitext = l2
		end
	end
	
	local kana
	local romaji
	
	local headword_template = find_headword_template(wikitext)
	if headword_template then
		for glob in mw.text.gsplit(headword_template:gsub('%}%}', '|'), '|') do
			if match(glob, '^[%s%.%-%^' .. kana_pattern .. '、]+$') then
				kana = glob; break
			end
		end
		romaji = headword_template:match("%|rom%=([^%|%}]+)")
	end
	
	if not kana and match(title, '^[' .. kana_pattern .. '、]+$') then
		kana = title
	end
	
	if not kana then
		if _wikitext then
			return nil
		else
			local main_entry = wikitext:match('{{ja%-see|(.-)[|}]')
			if main_entry then
				local new_wikitext = export.get_etym_section(main_entry, title)
				return export.fetch_reading(main_entry, new_wikitext:gsub('===ရင်းမြစ် %d+===', ''))
			else
				return nil
			end
		end
	end
	if not romaji then
		if headword_template:find('ja%-verb%|') then
			kana = gsub(kana, 'う$', '.う')
		elseif headword_template:find('ja%-adj') and (headword_template:find('%|infl=i') or headword_template:find('%|infl=い') or headword_template:find('%|decl=i') or headword_template:find('%|decl=い')) then
			kana = gsub(kana, 'い$', '.い')
		elseif headword_template:find('proper') then
			kana = gsub(kana, '^', '^')
			kana = gsub(kana, ' ', ' ^')
			kana = gsub(kana, '%-', '-^')
		end
		romaji = m_ja.kana_to_romaji(kana)
	end
	
	kana = gsub(kana, '[%s%.%-%^]', '')
	return kana, romaji
	
end
]]
return export