Module:PsalmTable

Documentation for this module may be created at Module:PsalmTable/doc

local vc = require('Module:VerseCalculations')
local pf = require('Module:PsalmFunctions')
local p = {}
local chapter=-1
local DEBUG
local debugLog = {}
local includeColors = true
local groupEnglish = true
local clusterCBC = true
local sectionInfo, subsectionInfo, speakerInfo, addresseeInfo, emotionInfo = {}, {}, {}, {}, {}
local includeSectionBreaks = false
local lastSectionLineSet = {}
local lastSubSectionLineSet = {}
local annotationList = {}
local colorMap = {}
local currentSectionColor
local colorSections = false

-- Fixes \n from SMW being passed as literal text
local function unescapeNewlines(s)
	return s:gsub("\\n", "\n")
	
end

local function normalizeV(verse)
    -- Ensure the input is a string
    if type(verse) ~= "string" or verse == "" then
        return -1
    end

    -- Extract base number and optional letter suffix
    local baseNumber, letters = verse:match("^(%d+)([a-z]*)$")
    if not baseNumber then
        return -1
    end

    -- Convert base number
    baseNumber = tonumber(baseNumber) or 0

    -- Add scaled value for letter suffix (e.g., "a" = 0.1, "b" = 0.2, ...)
    if letters ~= "" then
        local letterValue = 0
        for i = 1, #letters do
            local char = letters:sub(i, i)
            if char:match("[a-z]") then
                letterValue = letterValue + (char:byte() - string.byte("a") + 1) * (0.1 ^ i)
            else
                return "Error: Invalid letter in verse suffix"
            end
        end
        baseNumber = baseNumber + letterValue
    end

    return baseNumber
end
function p.normalizeVerse(frame)
	local verse = frame.args[1]

	return normalizeV(verse)
end

local function parseLineID(line)
	if type(line) ~= "string" then return nil end
	local number, letter = line:match("^(%d+)([a-z]?)$")
	if not number then return nil end
	return tonumber(number), letter or ""
end

--[[
local function compareLines(anum, aletter, bnum, bletter)
	if not anum or not bnum then return false end  -- guard against nil

	if anum ~= bnum then
		return anum < bnum
	end
	return aletter < bletter
end
]]
-- return a numeric 
local function compareLines(anum, aletter, bnum, bletter)
	if not anum or not bnum then return 0 end
	if anum ~= bnum then
		return anum < bnum and -1 or 1
	end
	if aletter == bletter then
		return 0
	end
	return aletter < bletter and -1 or 1
end

--[[
local function isLineInRange(lineID, firstLine, lastLine)
	local linenum, lineletter = parseLineID(lineID)
	local firstnum, firstletter = parseLineID(firstLine)
	local lastnum, lastletter = parseLineID(lastLine)

	if not linenum or not firstnum or not lastnum then
		return false
	end

	return compareLines(firstnum, firstletter, linenum, lineletter)
	       and compareLines(linenum, lineletter, lastnum, lastletter)
end
]]
local function isLineInRange(lineID, firstLine, lastLine)
	local linenum, lineletter = parseLineID(lineID)
	local firstnum, firstletter = parseLineID(firstLine)
	local lastnum, lastletter = parseLineID(lastLine)

	if not linenum or not firstnum or not lastnum then
		return false
	end

	-- Check: firstLine ≤ lineID ≤ lastLine
	return compareLines(firstnum, firstletter, linenum, lineletter) <= 0 and
	       compareLines(linenum, lineletter, lastnum, lastletter) <= 0
end

local function makeSubobjectString(data, lineID)
	local wordID = data.WordID or ""
	local parts = mw.text.split(wordID, "-")
	local writtenID = parts[2] or ""
	local lexemeID = parts[3] or ""

	local lineIDStr = data.LineID or ""
	local num, letter = parseLineID(lineIDStr)

	local verse = ""
	local clause = ""

	-- If lineID looks like "1.2", extract "1" as verse and "2" as clause
	local v, c = lineIDStr:match("^(%d+)%.(%d+)$")
	if v and c then
		verse = v
		clause = c
	else
		verse = num or ""
	end

	local normalizedVerse = normalizeV(verse) * 10

	-- Normalize Split: ensure it's a string ("true" or "")
	local isSplit = (data.Split == true or data.Split == "true") and "true" or ""

	if DEBUG and isSplit then 
		table.insert(debugLog, "Word " .. wordID .. " has Split:" .. tostring(isSplit)) 
	end

	return string.format(
		'{{#subobject:|IsCategory=TextualWord|WordID=%s|Split=%s|WrittenWordID=%s|LexemeID=%s|LineID=%s|Verse=%s|Clause=%s|NormalizedVerse=%s|Language=%s|Word=%s|Index=%s}}',
		wordID,
		isSplit,
		writtenID,
		lexemeID,
		lineIDStr,
		verse,
		clause,
		normalizedVerse,
		data.Language or "",
		data.Word or "",
		data.Index or ""
	)
end



local function adjustRowspansForBreaks()
	-- Clear previous sets
	lastSectionLineSet = {}
	lastSubSectionLineSet = {}

	-- Track section last lines
	for _, sec in ipairs(sectionInfo) do
		if sec.LastLine then
			lastSectionLineSet[sec.LastLine] = true
		end
	end

	-- Track subsection last lines only if not already in section
	for _, sec in ipairs(subsectionInfo) do
		if sec.LastLine and not lastSectionLineSet[sec.LastLine] then
			lastSubSectionLineSet[sec.LastLine] = true
		end
	end

	-- Combine both into one list for breaks
	local breakLineIDs = {}
	for lineID in pairs(lastSectionLineSet) do
		table.insert(breakLineIDs, lineID)
	end
	for lineID in pairs(lastSubSectionLineSet) do
		table.insert(breakLineIDs, lineID)
	end

	-- Sort: numeric prefix, alpha suffix
	table.sort(breakLineIDs, function(a, b)
		local numA, suffixA = a:match("^(%d+)(%a*)$")
		local numB, suffixB = b:match("^(%d+)(%a*)$")

		numA = tonumber(numA) or 0
		numB = tonumber(numB) or 0

		if numA == numB then
			return suffixA < suffixB
		else
			return numA < numB
		end
	end)

	-- Count how many breaks fall within a given section range
	local function countBreaksInRange(firstLine, lastLine)
		local count = 0
		for _, lineID in ipairs(breakLineIDs) do
			if isLineInRange(lineID, firstLine, lastLine) then
				count = count + 1
			end
		end
		return count
	end

	-- Adjust speaker sections
	for _, sec in ipairs(speakerInfo) do
		local base = tonumber(sec.LineCount) or 1
		local add = countBreaksInRange(sec.FirstLine, sec.LastLine)
		sec.LineCount = base + add
	end

	-- Adjust addressee sections
	for _, sec in ipairs(addresseeInfo) do
		local base = tonumber(sec.LineCount) or 1
		local add = countBreaksInRange(sec.FirstLine, sec.LastLine)
		sec.LineCount = base + add
	end

	return nil
end

local function renderWordSpan(w)
	--table.insert(debugLog, "ℹ️ Raw WordID: [" .. tostring(w.WordID) .. "]")
	local wordID = mw.text.trim(w.WordID or "")
	local wordIDs = mw.text.split(wordID, " ")
	
	local allIDs = {}
	for _, id in ipairs(wordIDs) do
		if id ~= "" then
			local safeID = id:gsub("%.", "_") -- replace . with _
			table.insert(allIDs, "id-" .. safeID)
		end
	end

	local firstSafeID = wordIDs[1] and wordIDs[1]:gsub("%.", "_") or ""
	local fullID = "id-" .. firstSafeID
	local participant = annotationList[fullID]

	local color = participant and colorMap[participant] or nil
	local styleAttr = color and (' style="background-color:' .. color .. ';"') or ""

	local class = (w.Language or "unknown"):lower()
	if w.Split then 
		class = class .. " split-word" 
	end
	
	if w.Word and w.Word:find("<span") then
		table.insert(debugLog, "⚠️ Word already contains span: " .. w.Word)
	end
	
	return string.format('<span class="%s %s"%s>%s</span>', class, table.concat(allIDs, " "), styleAttr, w.Word or "")
end

local function sortLineIDsFromSections(sections)
	local lineIDSet = {}
	for _, sec in ipairs(sections) do
		if sec.FirstLine and sec.LastLine then
			lineIDSet[sec.FirstLine] = true
			lineIDSet[sec.LastLine] = true
		end
	end
	return sortLineIDs(lineIDSet)
end

local function getSubsections(sectionData)
	local subs = {}
	for _, s in ipairs(sectionData) do
		if s.Column == "Subsection" then
			table.insert(subs, s)
		end
	end
	-- sort by FirstLine
	table.sort(subs, function(a, b)
		local anum, aletter = parseLineID(a.FirstLine)
		local bnum, bletter = parseLineID(b.FirstLine)
		if anum ~= bnum then
			return anum < bnum
		else
			return aletter < bletter
		end
	end)
	return subs
end

local function findSubsectionForLine(lineID, subsections)
	for _, sub in ipairs(subsections) do
		local sFirstNum, sFirstLet = parseLineID(sub.FirstLine)
		local sLastNum, sLastLet = parseLineID(sub.LastLine)
		local lNum, lLet = parseLineID(lineID)

		-- lineID is within subsection range if:
		--   lineID >= FirstLine  ⇨  not (lineID < FirstLine)
		--   lineID <= LastLine   ⇨  not (lineID > LastLine)
		local lineIsAfterOrAtStart = not compareLines(lNum, lLet, sFirstNum, sFirstLet)
		local lineIsBeforeOrAtEnd  = not compareLines(sLastNum, sLastLet, lNum, lLet)

		if lineIsAfterOrAtStart and lineIsBeforeOrAtEnd then
			return sub
		end
	end
	return nil
end
local function buildGenericSectionCell(lineID, column, entries)
	if not entries or #entries == 0 then
		return nil -- no sections at all
	end

	for _, sec in ipairs(entries) do
		if lineID == sec.FirstLine then
			local rowspan = tonumber(sec.LineCount) or 1
			local value = sec.Heading or ""
			local styleAttr = ""
			if includeColors and sec.Color then
				styleAttr = string.format(' style="background-color: %s;"', tostring(sec.Color))
			end

			local classList = { tostring(column:lower()) .. "-bar" }
			if sec.Annotation then
				table.insert(classList, tostring(column:lower() .. '-' .. sec.Annotation:lower()))
			end
			local classAttr = table.concat(classList, " ")

			return string.format(
				'|%s class="%s" data-line="%s" section-type="%s" rowspan=%d | <div>%s</div>',
				styleAttr,
				classAttr,
				tostring(lineID),
				tostring(column),
				rowspan,
				tostring(value)
			)
		elseif isLineInRange(lineID, sec.FirstLine, sec.LastLine) then
			return nil -- skip line, part of a rowspan
		end
	end

	-- If there are entries, but no match for this lineID, return an empty cell
	return '| '
end


local readyForSubsection = nil
local oneRowSkipped = nil

local function buildSubsectionCell(lineID)
	for _, sec in ipairs(subsectionInfo) do
		if lineID == sec.FirstLine then
			-- first is the heading, then prepare the content for next time
			local rowspan = tonumber(sec.LineCount) or 1

			local subHeading = sec.Heading or ""
			local sectionHeading = ""
			local roomToBreathe = tonumber(rowspan) > 4

			if mw.ustring.match(subHeading, "^File:.+%.%a%a%a$") then
				subHeading = string.format('[[%s|50px|frameless|center|class=aag-icon]]', subHeading)
			end
			subHeading = '<div class="subsection-content" data-rowcount="' .. rowspan .. '">' .. 
			subHeading .. "</div>"

			for _, parent in ipairs(sectionInfo or {}) do
				--if isLineInRange(sec.FirstLine, parent.FirstLine, parent.LastLine) then
				if sec.FirstLine == parent.FirstLine then
					sectionHeading = '<div class="section-heading">' .. (parent.Heading or "") .. "</div>"
					break
				end
			end

			local styleAttr = ""
			if includeColors and sec.Color then
				styleAttr = string.format(' style="background-color: %s;"', tostring(sec.Color))
			end

			if sectionHeading ~= "" then
				-- need to handle section before subsection
				
				if roomToBreathe then
					readyForSubsection = {
						style = styleAttr,
						content = subHeading,
						rowspan = rowspan - 2
					}
		
					return string.format(
						'|%s class="subsection-bar" data-line="%s" section-type="Subsection" rowspan=2 | %s',
						styleAttr,
						tostring(lineID),
						sectionHeading
					)
				else				
					readyForSubsection = {
						style = styleAttr,
						content = subHeading,
						rowspan = rowspan - 1
					}
					readyForSubsection = nil
		
					return string.format(
						'|%s class="subsection-bar" data-line="%s" section-type="Subsection" | %s',
						styleAttr,
						tostring(lineID),
						sectionHeading
					)
				end
			else
				-- just the subsection, right now
					readyForSubsection = {
						style = styleAttr,
						content = subHeading,
						rowspan = rowspan
					}
				-- handle subsection content
				local subsectionContent = string.format(
					'|%s class="subsection-bar" data-line="%s" section-type="Subsection" rowspan=%d | %s',
					readyForSubsection.style or '',
					tostring(lineID),
					readyForSubsection.rowspan or 1,
					readyForSubsection.content or ''
				)
				readyForSubsection = nil
				return subsectionContent
			end
				
		elseif readyForSubsection then
			if not oneRowSkipped then
				-- must skip one row
				oneRowSkipped = true
				return nil
			else
				-- reset this for the future
				oneRowSkipped = nil
				
				-- handle subsection content
				local subsectionContent = string.format(
					'|%s class="subsection-bar" data-line="%s" section-type="Subsection" rowspan=%d | %s',
					readyForSubsection.style or '',
					tostring(lineID),
					readyForSubsection.rowspan or 1,
					readyForSubsection.content or ''
				)
				readyForSubsection = nil
				return subsectionContent
			end

		elseif isLineInRange(lineID, sec.FirstLine, sec.LastLine) then
			-- no cell at all, since rowspan covers this
			return nil
		end
	end

	return '| '
end


local function buildEmotionCell(lineID)
	local sources = { subsectionInfo, sectionInfo }

	for _, source in ipairs(sources) do
		for _, sec in ipairs(source) do
			if lineID == sec.FirstLine and sec.Emotion then
				local rowspan = tonumber(sec.LineCount) or 1
				local value = sec.Emotion or "none specified"
				local styleAttr = ""
				if includeColors and sec.Color then
					styleAttr = string.format(' style="background-color: %s;"', tostring(sec.Color))
				end
				
				return string.format(
					'|%s class="emotion-bar" data-line="%s" section-type="Emotion" rowspan=%d | <div>%s</div>',
					styleAttr,
					tostring(lineID),
					rowspan,
					tostring(value)
				)


			elseif isLineInRange(lineID, sec.FirstLine, sec.LastLine) and sec.Emotion then
				-- emotion already entered early; leave blank
				return nil
			end
		end
	end


		-- If there are entries, but no match for this lineID, return an empty cell
	return '| '

end


local function buildSectionCell(lineID, column)
	
	-- General-purpose columns
	local infoMap = {
		Speaker = speakerInfo,
		Section = sectionInfo,
		Addressee = addresseeInfo,
	}
	local entries = infoMap[column]
	if not entries then return nil end
	return buildGenericSectionCell(lineID, column, entries)

end

local function sortSectionListByFirstLine(sections)
	table.sort(sections, function(a, b)
		local anum, aletter = parseLineID(a.FirstLine)
		local bnum, bletter = parseLineID(b.FirstLine)
		return compareLines(anum, aletter, bnum, bletter) < 0
	end)
end
local function getSectionAnnotationForLine(lineID, sectionInfo)
	local lNum, lLet = parseLineID(lineID)
	for _, sec in ipairs(sectionInfo) do
		local fNum, fLet = parseLineID(sec.FirstLine)
		local tNum, tLet = parseLineID(sec.LastLine)

		if compareLines(fNum, fLet, lNum, lLet) <= 0 and
		   compareLines(lNum, lLet, tNum, tLet) <= 0 then
			return sec.Annotation or nil
		end
	end
	if DEBUG then table.insert(debugLog, "⚠️ No section annotation match for lineID: " .. tostring(lineID)) end
	return nil
end


local function getSectionNumberForLine(lineID, info)
	local lNum, lLet = parseLineID(lineID)
	for i, sec in ipairs(info) do
		local fNum, fLet = parseLineID(sec.FirstLine)
		local tNum, tLet = parseLineID(sec.LastLine)

		if compareLines(fNum, fLet, lNum, lLet) <= 0 and
		   compareLines(lNum, lLet, tNum, tLet) <= 0 then
			return i
		end
	end
	-- table.insert(debugLog, "⚠️ No section match for lineID: " .. tostring(lineID)) end
		-- If there are entries, but no match for this lineID, return an empty cell
	return nil

end
local function groupAdjacentWordsByWordID(wordList)
	if not groupEnglish then return end
	local result = {}

	-- Step 1: Sort by Index
	table.sort(wordList, function(a, b)
		return (tonumber(a.Index) or 0) < (tonumber(b.Index) or 0)
	end)

	-- Step 2: Group only consecutive words that share the same WordID
	local currentGroup = nil
	for _, w in ipairs(wordList) do
		if currentGroup and currentGroup.WordID == w.WordID then
			if not w.Split then
				currentGroup.Word = currentGroup.Word .. " "
			end
			currentGroup.Word = currentGroup.Word .. (w.Word or "")
		else
			if currentGroup then
				table.insert(result, currentGroup)
			end
			currentGroup = {
				WordID = w.WordID,
				Language = w.Language,
				Word = w.Word or ""
			}
		end
	end

	if currentGroup then
		table.insert(result, currentGroup)
	end

	return result
end


local function buildHebrewCell(lineID, hebrewWords)
	local hebrewSpans = {}
	local prevMid = nil

	for _, h in ipairs(hebrewWords or {}) do
		local parts = mw.text.split(h.WordID or "", "-")
		local mid = parts[2]
		if prevMid and mid ~= prevMid then
			table.insert(hebrewSpans, " ")
		end
		prevMid = mid
		table.insert(hebrewSpans, renderWordSpan(h))
	end

	local result = string.format(
		'<div class="line" data-line="%s">%s</div>',
		lineID,
		table.concat(hebrewSpans, ""))
	
	table.insert(debugLog, "ℹ️ Hebrew for line " .. lineID .. ": " .. result)

	return result
	
end


local function restoreInsertions(englishWords, insertions, currentLineID)
	local englishSpans = {}
	local insertQueue = {}

	-- Filter insertions for this line
	for _, ins in ipairs(insertions or {}) do
		if ins.lineID == currentLineID then
			table.insert(insertQueue, ins)
		end
	end

	-- Sort insertions by index
	table.sort(insertQueue, function(a, b)
		return (tonumber(a.index or 0) or 0) < (tonumber(b.index or 0) or 0)
	end)

	table.insert(debugLog, 'ℹ️️ Debug: insertions for line ' .. currentLineID .. ' = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(insertQueue)) .. '</pre></div>')

	local insertIndex = 1
	local ins = insertQueue[insertIndex]
	local accumulatedText = ""
	local wordCount = 0

	for _, word in ipairs(englishWords or {}) do
		local plain = word.Word or ""
		local rendered = renderWordSpan(word)
		local inserted = false

		while ins do
			local targetText = ins.textBefore or ""

			if targetText == "" then
				-- We'll insert this at the beginning after loop
				break
			end

			local candidate = (accumulatedText .. " " .. plain):gsub("^%s*", ""):gsub("%s*$", ""):gsub("%s+", " ")

			if candidate == targetText then
				-- Perfect match: insert after this word
				table.insert(englishSpans, rendered)
				table.insert(englishSpans,
					string.format('<span class="insertion" data-index="%d">%s</span>',
						wordCount,
						mw.text.nowiki(ins.text or "⟪insert⟫")
					)
				)
				accumulatedText = candidate
				inserted = true
				insertIndex = insertIndex + 1
				ins = insertQueue[insertIndex]
				table.insert(debugLog, 'ℹ️ Debug: Found a perfect match for <pre>' .. targetText .. '</pre>')
				break

			elseif targetText == candidate:sub(1, #targetText) and #targetText > #accumulatedText then
				
				table.insert(debugLog, 'ℹ️ Debug: need to split spans because <pre>' .. targetText .. '</pre> demands it')

				
				-- Partial match: split current span
				local matchLength = #targetText - #accumulatedText
				local beforePart = plain:sub(1, matchLength)
				local afterPart = plain:sub(matchLength + 1)

				local spanBase = string.format('class="english" data-wordid="%s"', mw.text.nowiki(word.WordID or ""))

				table.insert(englishSpans, string.format('<span %s>%s</span>', spanBase, mw.text.nowiki(beforePart)))
				table.insert(englishSpans,
					string.format('<span class="insertion" data-index="%d">%s</span>',
						wordCount,
						mw.text.nowiki(ins.text or "⟪insert⟫")
					)
				)
				table.insert(englishSpans, string.format('<span %s>%s</span>', spanBase, mw.text.nowiki(afterPart)))

				accumulatedText = accumulatedText .. plain
				inserted = true
				insertIndex = insertIndex + 1
				ins = insertQueue[insertIndex]
				break
			else
				table.insert(debugLog, '⚠️ Debug: No match possible for <pre>' .. targetText .. '</pre> and <pre>' .. candidate .. '</pre>')

				-- No match
				break
			end
		end

		if not inserted then
			table.insert(englishSpans, rendered)
			accumulatedText = accumulatedText .. " " .. plain
		end

		wordCount = wordCount + 1
	end

	-- Handle any remaining insertions
	while ins do
		if (ins.textBefore or "") == "" then
			-- Insert at beginning
			table.insert(englishSpans, 1,
				string.format('<span class="insertion" data-index="0">%s</span>',
					mw.text.nowiki(ins.text or "⟪insert⟫")
				)
			)
			table.insert(debugLog, 'ℹ️️ Debug: Beginning insertion <pre>' .. ins.text .. '</pre>')
		else
			-- Insert at end
			table.insert(englishSpans,
				string.format('<span class="insertion" data-index="%d">%s</span>',
					wordCount,
					mw.text.nowiki(ins.text or "⟪insert⟫")
				)
			)
			table.insert(debugLog, 'ℹ️️ Debug: Ending insertion <pre>' .. ins.text .. '</pre>')
		end
		insertIndex = insertIndex + 1
		ins = insertQueue[insertIndex]
	end

	return englishSpans
end





local function buildLineRow(lineID, group, insertions, lastLineID, groupBy, buildSubobjects, subobjects, putVerseFirst)
	local hebrewSpans, englishSpans = {}, {}
	local prevMid = nil

	for _, h in ipairs(group.hebrew or {}) do
		local parts = mw.text.split(h.WordID or "", "-")
		local mid = parts[2]
		if prevMid and mid ~= prevMid then
			table.insert(hebrewSpans, " ")
		end
		prevMid = mid
		table.insert(hebrewSpans, renderWordSpan(h))
	end
	
	if DEBUG then
		table.insert(debugLog, 'ℹ️ Debug: hebrew = <div class="mw-collapsible mw-collapsed"><pre>' .. mw.text.nowiki(mw.dumpObject(hebrewSpans)) .. '</pre></div>')
	end


	if DEBUG then table.insert(debugLog, "ℹ️️ Debug: buildLineRow: " .. lineID) end
	
	local groupedEnglish = group.english
	if clusterCBC and #insertions == 0 then
		groupedEnglish = groupAdjacentWordsByWordID(group.english)
	end

	if DEBUG then
		table.insert(debugLog, 'ℹ️ Debug: cluster CBC = ' .. tostring(clusterCBC))
		table.insert(debugLog, 'ℹ️️ Debug: english = <div class="mw-collapsible mw-collapsed"><pre>' .. mw.text.nowiki(mw.dumpObject(groupedEnglish)) .. '</pre></div>')
	end

	englishSpans = restoreInsertions(groupedEnglish, insertions, lineID)


	
	local hasSubsections = type(subsectionInfo) == "table" and #subsectionInfo > 0
	local row = {}
	local subobjectRow = {}

	local sectionNum = getSectionNumberForLine(lineID, sectionInfo)
	if groupBy=="sections" and sectionNum then
		local sectionClass = "section-" .. (sectionNum or "unknown")
		
		if sectionClass == "section-unknown" and DEBUG then
			table.insert(debugLog, "No section class for line " .. tostring(lineID))
		end
		
		-- Look for a background color
		for _, sec in ipairs(subsectionInfo or {}) do
			if lineID == sec.FirstLine then
				if includeColors and sec.Color then
					currentSectionColor = string.format(' style="background-color: %s;"', tostring(sec.Color))
					break
				end
			end
		end
		
		if colorSections and currentSectionColor then
			-- start new row with background color
			table.insert(row, '|- class="' .. sectionClass .. '"' .. currentSectionColor)
		else
			-- start new row without background color
			table.insert(row, '|- class="' .. sectionClass .. '"')
		end

		
	elseif groupBy=="speakers" then
		local speaker = getSectionAnnotationForLine(lineID, speakerInfo)
		local speakerClass = "speaker-" .. (speaker or "unknown")
	
		if speakerClass == "" and DEBUG then table.insert(debugLog, "No speaker class for line " .. lineID) end

		-- start new row
		table.insert(row, '|- class="' .. speakerClass .. '"')		
	else
		-- start new row with no class
		table.insert(row, '|- ')
	end
	
	-- speaker
	if #speakerInfo > 0 then
		table.insert(row, buildSectionCell(lineID, "Speaker"))
	end
	
	if hasSubsections and #subsectionInfo > 0 then
			table.insert(row, buildSubsectionCell(lineID))
	elseif #sectionInfo > 0 then
		table.insert(row, buildSectionCell(lineID, "Section"))
	end 

	if putVerseFirst then
		table.insert(row, string.format('| class="verse-cell" | %s', lineID))
	end
	
	local num, letter = parseLineID(lineID)

	
	table.insert(row, '| dir="rtl" class="hebrew-cell" | ' .. buildHebrewCell(lineID, group.hebrew, buildSubobjects, subobjects))
	table.insert(subobjectRow, string.format(
			'{{#subobject:|IsCategory=RenderedHebrew|Chapter=%s|Verse=%s|LineID=%s|IsRenderedHebrewFor=%s|NormalizedVerse=%s|LineData=%s}}',
				chapter or "999",
				num or "999",
				lineID or "999",	
				"Data/" .. chapter .. "." .. lineID,
				normalizeV(lineID or "999") * 10,
				buildHebrewCell(lineID, group.hebrew, buildSubobjects, subobjects):gsub("|", "{{!}}")
			))

	if putVerseFirst == false then
		table.insert(row, string.format('| class="verse-cell" | %s', lineID))
	end
	
	--table.insert(row, string.format('| class="cbc-cell" | <div class="line" data-line="%s">%s</div>', 
	--	lineID, table.concat(englishSpans, " ")))
	
	local englishLine = ""
	for i, span in ipairs(englishSpans) do
		if i > 1 then
			local isSplit = string.find(span, 'class="[^"]*split%-word')
			if not isSplit then
				englishLine = englishLine .. " "
			end
		end
		englishLine = englishLine .. span
	end
	
	table.insert(row, string.format('| class="cbc-cell" | <div class="line" data-line="%s">%s</div>', 
		lineID, englishLine))
	
	
	
	table.insert(subobjectRow, string.format(
			'{{#subobject:|IsCategory=RenderedCBC|Chapter=%s|Verse=%s|LineID=%s|IsRenderedCBCFor=%s|NormalizedVerse=%s|LineData=%s}}',
				chapter or "999",
				num or "999",
				lineID or "999",	
				"Data/" .. chapter .. "." .. lineID,
				normalizeV(lineID or "999") * 10,
				table.concat(englishSpans, " "):gsub("|", "{{!}}")
			))

	if #sectionInfo > 0 then
		table.insert(row, buildEmotionCell(lineID))
	end 
	if #addresseeInfo > 0 then
		table.insert(row, buildSectionCell(lineID, "Addressee"))
	end 

	if includeSectionBreaks and lastSectionLineSet and lastSectionLineSet[lineID] and lineID ~= lastLineID then
		-- new row with one cell of colspan=4
		table.insert(row, '|- class="section-break"')
		table.insert(row, '| colspan="4" |')
	elseif includeSectionBreaks and lastSubSectionLineSet and lastSubSectionLineSet[lineID] and lineID ~= lastLineID then
		-- new row with one cell of colspan=4
		table.insert(row, '|- class="subsection-break"')
		table.insert(row, '| colspan="4" |')
	end


	return table.concat(row, "\n"), table.concat(subobjectRow, "\n")
	--return table.concat(row, "\n"):gsub("\n+$", "")

end

local function parseOverlayLines(content, buildSubobjects)
	local lines, subobjects = {}, {}
	local decoded

	-- Try to parse the whole content as JSON
	local isJSON = content:match("^%s*[%[{]")
	if isJSON then
		local ok, result = pcall(mw.text.jsonDecode, content)
		if ok and type(result) == "table" then
			decoded = result
		end
	end

	-- If whole-content JSON parsed successfully, treat it as a JSON array
	if decoded then
		for _, data in ipairs(decoded) do
			local lang = data.Language and data.Language:lower()
			local lineID = data.LineID
			if lang and lineID then
				lines[lineID] = lines[lineID] or {hebrew = {}, english = {}}
				table.insert(lines[lineID][lang], data)

				if buildSubobjects then
					table.insert(subobjects, makeSubobjectString(data, lineID))
				end
			end
		end
	else
		-- Otherwise, fall back to line-by-line parsing
		for line in content:gmatch("[^\r\n]+") do
			local ok, data = pcall(mw.text.jsonDecode, line)
			if ok and data then
				local lang = data.Language and data.Language:lower()
				local lineID = data.LineID
				if lang and lineID then
					lines[lineID] = lines[lineID] or {hebrew = {}, english = {}}
					table.insert(lines[lineID][lang], data)

					if buildSubobjects then
						table.insert(subobjects, makeSubobjectString(data, lineID))
					end
				end
			end
		end
	end

	return lines, subobjects
end
local function sortLineIDs(lines)
	local ids = {}
	for lineID in pairs(lines) do
		table.insert(ids, lineID)
	end

	table.sort(ids, function(a, b)
		-- Case 1: "major.minor" numeric format
		local amaj, amin = a:match("^(%d+)%.(%d+)$")
		local bmaj, bmin = b:match("^(%d+)%.(%d+)$")
		if amaj and bmaj then
			amaj, amin = tonumber(amaj), tonumber(amin)
			bmaj, bmin = tonumber(bmaj), tonumber(bmin)
			if amaj ~= bmaj then return amaj < bmaj end
			return amin < bmin
		end

		-- Case 2: "majorletter" format like "1a", "10b"
		local anum, aletter = a:match("^(%d+)([a-zA-Z])$")
		local bnum, bletter = b:match("^(%d+)([a-zA-Z])$")
		if anum and bnum then
			anum = tonumber(anum)
			bnum = tonumber(bnum)
			if anum ~= bnum then return anum < bnum end
			return aletter < bletter
		end

		-- Fallback: string comparison
		return a < b
	end)

	return ids
end


local function sortWordGroupsByIndex(lines)
	for _, group in pairs(lines) do
		table.sort(group.hebrew, function(a, b) return a.Index < b.Index end)
		table.sort(group.english, function(a, b) return a.Index < b.Index end)
	end
end

local function cleanSMWInput(raw)
	raw = unescapeNewlines(raw or "")
	raw = raw:gsub("%[%[SMW::off%]%]", ""):gsub("%[%[SMW::on%]%]", "")
	return raw
end

local function parseJSONArg(frame, key)
	local raw = frame.args[key] or "{}"
	raw = raw:gsub("\\n", "")
	local ok, result = pcall(mw.text.jsonDecode, raw)
	if ok and type(result) == "table" then
		return result
	else
		return {}
	end
end

local function extractSectionColumns(sections)
	--local speakerInfo, sectionInfo, subsectionInfo, addresseeInfo, emotionInfo = {}, {}, {}, {}, {}

	for _, sec in ipairs(sections) do
		local col = sec.Column
		if col == "Speaker" then
			table.insert(speakerInfo, sec)
		elseif col == "Section" then
			table.insert(sectionInfo, sec)
		elseif col == "Subsection" then
			table.insert(subsectionInfo, sec)
		elseif col == "Addressee" then
			table.insert(addresseeInfo, sec)
		elseif col == "Emotion" then
			table.insert(emotionInfo, sec)
		end
	end
	if DEBUG and #speakerInfo > 0 then
		
		table.insert(debugLog, 'ℹ️ Debug: section counts - Speaker=' .. #speakerInfo ..
			', Section=' .. #sectionInfo ..
			', Subsection=' .. #subsectionInfo ..
			', Addressee=' .. #addresseeInfo ..
			', Emotion=' .. #emotionInfo)

		table.insert(debugLog, 'ℹ️ Debug: speaker = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(speakerInfo)) .. '</pre></div>')
		table.insert(debugLog, 'ℹ️ Debug: sections = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sectionInfo)) .. '</pre></div>')
		table.insert(debugLog, 'ℹ️ Debug: subsections = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(subsectionInfo)) .. '</pre></div>')

	end
	return nil -- speakerInfo, sectionInfo, subsectionInfo, addresseeInfo, emotionInfo
end



function p.renderEnglishCellOLD(lineID, group)
	local englishSpans = {}
	local groupedEnglish = groupAdjacentWordsByWordID(group.english)
	for _, e in ipairs(groupedEnglish) do
		table.insert(englishSpans, renderWordSpan(e))
	end
	return string.format('| class="cbc-cell" | <div class="line" data-line="%s">%s</div>', 
		lineID, table.concat(englishSpans, " "))
end

function p.renderEnglishCell(lineID, group)
	local englishSpans = {}
	local groupedEnglish = groupAdjacentWordsByWordID(group.english)

	for i, e in ipairs(groupedEnglish) do
		local span = renderWordSpan(e)
		
		if i > 1 then
			local isSplit = string.find(span, 'class="[^"]*split%-word')
			if not isSplit then
				table.insert(englishSpans, " ") -- insert space
			end
		end

		table.insert(englishSpans, span)
	end

	return string.format(
		'| class="cbc-cell" | <div class="line" data-line="%s">%s</div>', 
		lineID, table.concat(englishSpans)
	)
end

local function cleanForJSON(str)
	-- Trim leading/trailing whitespace
	str = mw.text.trim(str or "")

	-- Remove raw control characters (ASCII 0–31 except \t (9) and \n (10) if needed)
	str = str:gsub("[%z\1-\8\11\12\14-\31]", "")

	-- Replace literal newlines inside quoted strings with \n
	str = str:gsub('"(.-)\n(.-)"', function(a, b)
		return '"' .. a .. '\\n' .. b .. '"'
	end)

	return str
end


function p.renderOverlayTableSection(frame)
	local out = {}
	local args = frame.args

	chapter = pf.getPsalmNumber(frame)

	-- Debug mode
	DEBUG = args["debug"] == "true"
	if DEBUG then table.insert(debugLog, "|}") end


	-- Extract and validate input text
	
	local content = nil
	local jsonPageList = args["jsonalignment"]
	
	if not jsonPageList or jsonPageList == "" then
		content = cleanSMWInput(args["text"] or args["tokens"])
		if content == "" then 
			return "" -- No content found
		end
	else
		-- Must wrap the page list string in a frame-like table for the function
		content = pf.concatJSONFromList({ args = { jsonPageList } })
	end


	-- Parse section JSON
	local sectionsJSON = cleanSMWInput(args["sections"] or "")
    sectionsJSON = cleanForJSON(sectionsJSON)
	
	local ok, sectionData = pcall(mw.text.jsonDecode, sectionsJSON)
	local sections = (ok and type(sectionData) == "table") and sectionData or {}
	table.insert(debugLog, 'ℹ️ sectionsJSON = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sectionsJSON)) .. '</pre></div>') 
	table.insert(debugLog, 'ℹ️ sectionData = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sectionData)) .. '</pre></div>') 

	-- Parse insertions JSON
	local insertionsJSON = cleanSMWInput(args["insertions"] or "")
	local ok, insertionsData = pcall(mw.text.jsonDecode, insertionsJSON)
	local insertions = (ok and type(insertionsData) == "table") and insertionsData or {}
	if #insertions > 0 then
		table.insert(debugLog, "Parsed insertions: " .. tostring(#insertions) .. " items")
		table.insert(debugLog, 'ℹ️ Insertions = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(insertions)) .. '</pre></div>') 
	 end

	-- Settings
	local groupBy = args["groupby"] or "sections"
	if groupBy then table.insert(debugLog, 'ℹ️ groupby=' .. tostring(groupBy)) end

	-- Section info
	--local speakerInfo, sectionInfo, subsectionInfo, addresseeInfo, emotionInfo = 
	extractSectionColumns(sections)
	sortSectionListByFirstLine(sectionInfo)
	includeSectionBreaks = #sectionInfo > 0
	table.insert(debugLog, 'ℹ️ Debug: sections = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sections)) .. '</pre></div>') 

	-- Line break tracking (optional)
	if includeSectionBreaks then
		adjustRowspansForBreaks()
	end
	
	-- Annotation and styling info
	annotationList = parseJSONArg(frame, "annotations") or {}
	table.insert(debugLog, 'ℹ️ Debug: annotations = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(annotationList)) .. '</pre></div>') 

	colorMap = parseJSONArg(frame, "colormap")
	table.insert(debugLog, 'ℹ️ Debug: colors = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(colorMap)) .. '</pre></div>') 
	local putVerseFirst = not (tostring(args["versefirst"] or ""):lower() == "false" or tostring(args["versefirst"]):lower() == "no")
	table.insert(debugLog, 'ℹ️ Debug: putVerseFirst=' .. tostring(putVerseFirst)) 

	clusterCBC = not (tostring(args["clusterCBC"] or ""):lower() == "false" or tostring(args["clusterCBC"]):lower() == "no")


	includeColors = not (tostring(args["colors"] or ""):lower() == "false" or tostring(args["colors"]):lower() == "no")
	-- if DEBUG then table.insert(debugLog, 'ℹ️ Debug: colors=' .. tostring(includeColors)) end

	groupEnglish = not (tostring(args["groupingEnglish"] or ""):lower() == "false" or tostring(args["groupingEnglish"]):lower() == "no")
	colorSections = not (tostring(args["colorsections"] or ""):lower() == "false" or tostring(args["colorsections"]):lower() == "no")

	-- Parse lines and optional SMW subobjects
	local buildSubobjects = args["subobjects"] or false
	local lines, subobjects = parseOverlayLines(content, buildSubobjects)
	sortWordGroupsByIndex(lines)
	table.insert(debugLog, 'ℹ️ Debug: lines = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(lines)) .. '</pre></div>') 

	-- Sort and iterate lines
	local sortedLineIDs = sortLineIDs(lines)
	local lastLineID = sortedLineIDs[#sortedLineIDs]
	local lineCount = 0

	for _, lineID in ipairs(sortedLineIDs) do
		local group = lines[lineID]
		lineCount = lineCount + 1
		if maxLines and lineCount > maxLines then break end

		local tableRow, subobjectRow = buildLineRow(
			lineID, group, insertions,
			lastLineID, groupBy, buildSubobjects, subobjects, putVerseFirst
		)

		table.insert(out, tableRow)
		if buildSubobjects then
			table.insert(subobjects, subobjectRow)
		end
	end

	-- Optional subobject output
	if buildSubobjects then
		frame:preprocess(table.concat(subobjects, "\n"))
	end

	if DEBUG then
		return table.concat(out, "\n") .. table.concat(debugLog, "\n")
	else
		return table.concat(out, "\n")
	end

end

function p.buildTableSubobjects(frame)
	local out = {}
	local args = frame.args

	chapter = pf.getPsalmNumber(frame)

	-- Debug mode
	DEBUG = args["debug"] == "true"
	if DEBUG then table.insert(debugLog, "|}") end


	-- Extract and validate input text
	local content = cleanSMWInput(args["text"] or args["tokens"])
	if content == "" then return "No content found." end

	-- Parse section JSON
	local sectionsJSON = cleanSMWInput(args["sections"] or "")
	local ok, sectionData = pcall(mw.text.jsonDecode, sectionsJSON)
	local sections = (ok and type(sectionData) == "table") and sectionData or {}
	table.insert(debugLog, 'ℹ️ Debug: sections = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sections)) .. '</pre></div>') 

	-- Parse insertions JSON
	local insertionsJSON = cleanSMWInput(args["insertions"] or "")
	local ok, insertionsData = pcall(mw.text.jsonDecode, insertionsJSON)
	local insertions = (ok and type(insertionsData) == "table") and insertionsData or {}
	if #insertions > 0 then
		table.insert(debugLog, "Parsed insertions: " .. tostring(#insertions) .. " items")
		table.insert(debugLog, 'ℹ️ Insertions = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(insertions)) .. '</pre></div>') 
	 end

	-- Settings
	local groupBy = args["groupby"] or "sections"
	if groupBy then table.insert(debugLog, 'ℹ️ groupby=' .. tostring(groupBy)) end

	-- Section info
	--local speakerInfo, sectionInfo, subsectionInfo, addresseeInfo, emotionInfo = 
	extractSectionColumns(sections)
	sortSectionListByFirstLine(sectionInfo)
	includeSectionBreaks = #sectionInfo > 0
	table.insert(debugLog, 'ℹ️ Debug: sections = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(sections)) .. '</pre></div>') 

	-- Line break tracking (optional)
	if includeSectionBreaks then
		adjustRowspansForBreaks()
	end
	
	-- Annotation and styling info
	annotationList = parseJSONArg(frame, "annotations") or {}
	table.insert(debugLog, 'ℹ️ Debug: annotations = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(annotationList)) .. '</pre></div>') 

	colorMap = parseJSONArg(frame, "colormap")
	table.insert(debugLog, 'ℹ️ Debug: colors = <div class=\"mw-collapsible mw-collapsed\"><pre>' .. mw.text.nowiki(mw.dumpObject(colorMap)) .. '</pre></div>') 
	local putVerseFirst = not (tostring(args["versefirst"] or ""):lower() == "false" or tostring(args["versefirst"]):lower() == "no")
	table.insert(debugLog, 'ℹ️ Debug: putVerseFirst=' .. tostring(putVerseFirst)) 

	clusterCBC = not (tostring(args["clusterCBC"] or ""):lower() == "false" or tostring(args["clusterCBC"]):lower() == "no")


	includeColors = not (tostring(args["colors"] or ""):lower() == "false" or tostring(args["colors"]):lower() == "no")
	-- if DEBUG then table.insert(debugLog, 'ℹ️ Debug: colors=' .. tostring(includeColors)) end

	groupEnglish = not (tostring(args["groupingEnglish"] or ""):lower() == "false" or tostring(args["groupingEnglish"]):lower() == "no")

	-- Parse lines and optional SMW subobjects
	local buildSubobjects = args["subobjects"] or false
	local lines, subobjects = parseOverlayLines(content, buildSubobjects)
	sortWordGroupsByIndex(lines)

	-- Sort and iterate lines
	local sortedLineIDs = sortLineIDs(lines)
	local lastLineID = sortedLineIDs[#sortedLineIDs]
	local lineCount = 0

	for _, lineID in ipairs(sortedLineIDs) do
		local group = lines[lineID]
		lineCount = lineCount + 1
		if maxLines and lineCount > maxLines then break end

		local tableRow, subobjectRow = buildLineRow(
			lineID, group, insertions,
			lastLineID, groupBy, buildSubobjects, subobjects, putVerseFirst
		)

			table.insert(subobjects, subobjectRow)
	end

	frame:preprocess(table.concat(subobjects, "\n"))

	if DEBUG then
		return table.concat(subobjects, "\n") .. table.concat(debugLog, "\n")
	else
		return table.concat(subobjects, "\n")
	end

end

function p.renderPsalmTableEscaped(frame)
  -- Call the actual rendering function with passed arguments
  local result = p.renderOverlayTableSection(frame)
  
  -- Replace all pipes with {{!}} to escape in wikitext
  local escaped = result:gsub("|", "{{!}}")

  return escaped
end
return p