-- Copyright 2017-2025 Mitchell. See LICENSE.
-- Unit tests for Scintillua lexers.

package.path = 'lexers/?.lua;' .. package.path

local lexer = require('lexer')
local word_match = lexer.word_match
lpeg = require('lpeg') -- not local for use by lexers in Lua 5.2+
-- Scintilla normally defines these.
lexer.FOLD_BASE, lexer.FOLD_HEADER, lexer.FOLD_BLANK = 0x400, 0x2000, 0x1000

-- Helper assert functions.

-- Asserts the given lexer contains the default Scintillua tags and Scintilla style names, and
-- that those items are correctly numbered. Scintillua tag numbers start at 1 while Scintilla
-- style numbers start at 33.
-- Note: the tag and style tables used are copied from lexer.lua since they are local to that file.
-- @param lex The lexer to style-check.
function assert_default_tags(lex)
	local default_tags = {
		'whitespace', 'comment', 'string', 'number', 'keyword', 'identifier', 'operator', 'error',
		'preprocessor', 'constant', 'variable', 'function', 'class', 'type', 'label', 'regex',
		'embedded'
	}
	for i = 1, #default_tags do
		local tag = default_tags[i]
		assert(lex._TAGS[tag], string.format("tag '%s' does not exist", tag))
		assert(lex._TAGS[tag] == i, 'default styles out of order')
	end
	local predefined_styles = {
		'default', 'line.number', 'brace.light', 'brace.bad', 'control.char', 'indent.guide',
		'call.tip', 'fold.display.text'
	}
	for i = 1, #predefined_styles do
		local style = predefined_styles[i]
		assert(lex._TAGS[style], string.format("style '%s' does not exist", style))
		assert(lex._TAGS[style] == i + 32, 'predefined styles out of order')
	end
	assert(lex._TAGS['whitespace.' .. lex._name]) -- auto-added by lexer.new()
end

-- Asserts the given lexer contains the given ordered list of rules.
-- @param lex The lexer to rule-check.
-- @param rules The ordered list of rule names the lexer should have.
function assert_rules(lex, rules)
	if rules[1] ~= 'whitespace' then table.insert(rules, 1, 'whitespace') end -- auto-added
	local j = 1
	for i = 1, #lex._rules do
		assert(lex._rules[rules[j]], string.format("rule '%s' does not exist", rules[j]))
		assert(lex._rules[i] == rules[j], string.format("'%s' ~= '%s'", lex._rules[i], rules[i] or ''))
		j = j + 1
	end
	if #lex._rules ~= #rules then error(string.format("'%s' rule not found", rules[j])) end
end

-- Asserts the given lexer contains the given set of extra tags in addition to its defaults.
-- @param lex The lexer to style-check.
-- @param tags The list of extra tag names the lexer should have.
function assert_extra_tags(lex, tags)
	for i = 1, #tags do
		assert(lex._TAGS[tags[i]], string.format("'%s' not found", tags[i]))
		assert(lex._extra_tags[tags[i]], string.format("'%s' not found", tags[i]))
	end
end

-- Asserts the given lexer contains the given set of child lexer names.
-- @param lex The lexer to child-check.
-- @param children The list of child lexer names the lexer should have.
function assert_children(lex, children)
	local j = 1
	for i = 1, #lex._CHILDREN do
		assert(lex._CHILDREN[i]._name == children[j],
			string.format("'%s' ~= '%s'", lex._CHILDREN[i]._name, children[j] or ''))
		j = j + 1
	end
	if #lex._CHILDREN ~= #children then error(string.format("child '%s' not found", children[j])) end
end

-- Asserts the given lexer produces the given tags after lexing the given code.
-- @param lex The lexer to use.
-- @param code The string code to lex.
-- @param expected_tags The list of expected tag names followed by the substring of code matched.
--   Whitespace tags are ignored for the sake of simplicity. Do not include them.
-- @param initial_style Optional current style. This is used for determining which language to
--   start in in a multiple-language lexer.
-- @usage assert_lex(lua, "print('hi')", {FUNCTION_BUILTIN, 'print', OPERATOR, '(', STRING, "'hi'",
--   OPERATOR, ')'}})
function assert_lex(lex, code, expected_tags, initial_style)
	if lex._lexer then lex = lex._lexer end -- note: lexer.load() does this
	local tags = lex:lex(code, initial_style or lex._TAGS['whitespace.' .. lex._name])
	local j = 1
	for i = 1, #tags, 2 do
		if not tags[i]:find('^whitespace') then
			local tag = tags[i]
			local text = code:sub(tags[i - 1] or 0, tags[i + 1] - 1)
			assert(tag == expected_tags[j] and text == expected_tags[j + 1], string.format(
				"('%s', '%s') ~= ('%s', '%s')", tag, text, expected_tags[j], expected_tags[j + 1]))
			j = j + 2
		end
	end
	if j - 1 ~= #expected_tags then
		error(string.format("('%s', '%s') not found", expected_tags[j], expected_tags[j + 1]))
	end
end

-- Asserts the given lexer produces the given fold points after lexing the given code.
-- @param lex The lexer to use.
-- @param code The string code to fold.
-- @param expected_fold_points The list of expected fold points from the lexer. Each fold point
--   is just a line number, starting from 1.
-- @param initial_style Optional current style. This is used for determining which language to
--   start in in a multiple-language lexer.
-- @return fold levels for any further analysis
-- @usage assert_fold_points(lua, "if foo then\n  bar\nend", {1})
function assert_fold_points(lex, code, expected_fold_points, initial_style)
	if lex._lexer then lex = lex._lexer end -- note: lexer.load() does this
	-- Since `M.style_at()` is provided by Scintilla and not available for tests, create it,
	-- using data from `lexer.lex()`.
	local tags = lex:lex(code, initial_style or lex._TAGS['whitespace.' .. lex._name])
	lexer.style_at = setmetatable({}, {
		__index = function(self, pos)
			for i = 2, #tags, 2 do if pos < tags[i] then return tags[i - 1] end end
		end
	})
	if not lexer.property then -- Scintilla normally creates this
		lexer.property, lexer.property_int = {}, setmetatable({}, {
			__index = function(t, k) return tonumber(lexer.property[k]) or 0 end,
			__newindex = function() error('read-only property') end
		})
	end
	lexer.property['fold'] = 1
	local levels = lex:fold(code, 1, lexer.FOLD_BASE)
	local j = 1
	for i = 1, #levels do
		local line_num = expected_fold_points[j]
		if i == line_num then
			assert(levels[i] >= lexer.FOLD_HEADER, string.format("line %i not a fold point", i))
			j = j + 1
		elseif line_num and i == -line_num then
			assert(levels[i] > levels[i + 1] & ~(lexer.FOLD_HEADER | lexer.FOLD_BLANK),
				string.format("line %i is not a fold end point", i))
			j = j + 1
		else
			assert(levels[i] <= lexer.FOLD_HEADER, string.format("line %i is a fold point", i))
		end
	end
	assert(j - 1 == #expected_fold_points, string.format("line %i is not a fold point", j))
	return levels
end

-- Unit tests.

function test_to_eol()
	local code = '#foo\\\nbar\\\nbaz'
	assert(lexer.to_eol('#'):match(code) == 6)
	assert(lexer.to_eol('#', true):match(code) == #code + 1)
	assert(not lexer.to_eol('f'):match(code))
	assert(lexer.to_eol():match(code) == 6)
end

function test_range()
	assert(lexer.range('"'):match('"foo\\"bar\n"baz') == 12)
	assert(lexer.range('"', true):match('"foo\\"bar\n"baz') == 10)
	assert(lexer.range('"', false, false):match('"foo\\"bar\n"baz') == 7)

	assert(lexer.range('(', ')'):match('(foo\\)bar)baz') == 7)

	assert(lexer.range('/*', '*/'):match('/*/*foo*/bar*/baz') == 10)
	assert(lexer.range('/*', '*/', false, false, true):match('/*/*foo*/bar*/baz') == 15)
end

function test_starts_line()
	assert(lexer.starts_line('#'):match('#foo') == 2)
	assert(lexer.starts_line('#'):match('\n#foo', 2) == 3)
	assert(not lexer.starts_line('#'):match(' #foo', 2))
	assert(lexer.starts_line('#', true):match(' #foo', 2) == 3)
	assert(lexer.starts_line('#', true):match('\t#foo', 2) == 3)
	assert(lexer.starts_line('#', true):match('\n\t#foo', 3) == 4)
	assert(not lexer.starts_line('#', true):match('\n.#foo', 3))
end

function test_after_set()
	local set = '=,'
	local regex = lexer.range('/')
	assert(lexer.after_set(set, regex):match('/foo'))
	assert(lexer.after_set(set, regex):match('/foo/'))
	assert(lexer.after_set(set, regex):match('foo=/bar/', 5))
	assert(lexer.after_set(set, regex):match('foo=\n/bar/', 6))
	assert(lexer.after_set(set, regex):match('foo, /bar/', 6))
	assert(not lexer.after_set(set, regex):match('foo/bar', 4))
	assert(not lexer.after_set(set, regex):match('foo / bar', 5))
	assert(lexer.after_set(set .. ' ', regex):match('foo / bar', 5))
end

function test_word_match()
	assert(word_match{'foo', 'bar', 'baz'}:match('foo') == 4)
	assert(not word_match{'foo', 'bar', 'baz'}:match('foo_bar'))
	assert(word_match({'foo!', 'bar?', 'baz.'}, true):match('FOO!') == 5)
	assert(word_match{'foo '}:match('foo')) -- spaces not allowed
	-- Test string list style.
	assert(word_match('foo bar baz'):match('foo') == 4)
	assert(not word_match('foo bar baz'):match('foo_bar'))
	assert(word_match('foo! bar? baz.', true):match('FOO!') == 5)
end

local DEFAULT = lexer.DEFAULT
local COMMENT = lexer.COMMENT
local STRING = lexer.STRING
local NUMBER = lexer.NUMBER
local KEYWORD = lexer.KEYWORD
local IDENTIFIER = lexer.IDENTIFIER
local OPERATOR = lexer.OPERATOR
local ERROR = lexer.ERROR
local PREPROCESSOR = lexer.PREPROCESSOR
local VARIABLE = lexer.VARIABLE
local FUNCTION = lexer.FUNCTION
local CLASS = lexer.CLASS
local TYPE = lexer.TYPE
local LABEL = lexer.LABEL
local REGEX = lexer.REGEX
local FUNCTION_BUILTIN = lexer.FUNCTION_BUILTIN
local CONSTANT_BUILTIN = lexer.CONSTANT_BUILTIN
local FUNCTION_METHOD = lexer.FUNCTION_METHOD
local TAG = lexer.TAG
local ATTRIBUTE = lexer.ATTRIBUTE
local VARIABLE_BUILTIN = lexer.VARIABLE_BUILTIN
local HEADING = lexer.HEADING
local BOLD = lexer.BOLD
local ITALIC = lexer.ITALIC
local UNDERLINE = lexer.UNDERLINE
local CODE = lexer.CODE
local LINK = lexer.LINK
local REFERENCE = lexer.REFERENCE
local ANNOTATION = lexer.ANNOTATION
local LIST = lexer.LIST

-- Tests a basic lexer with a few simple rules and no custom styles.
function test_basics()
	local lex = lexer.new('test')
	assert_default_tags(lex)
	lex:add_rule('keyword', lex:tag(KEYWORD, word_match('foo bar baz')))
	lex:add_rule('string', lex:tag(STRING, lexer.range('"')))
	lex:add_rule('number', lex:tag(NUMBER, lexer.integer))
	local code = [[foo bar baz "foo bar baz" 123]]
	local tags = {
		KEYWORD, 'foo', --
		KEYWORD, 'bar', --
		KEYWORD, 'baz', --
		STRING, '"foo bar baz"', --
		NUMBER, '123'
	}
	assert_lex(lex, code, tags)
end

-- Tests that lexer rules are added in an ordered sequence and that modifying rules in place
-- works as expected.
function test_rule_order()
	local lex = lexer.new('test')
	lex:add_rule('identifier', lex:tag(IDENTIFIER, lexer.word))
	lex:add_rule('keyword', lex:tag(KEYWORD, lpeg.P('foo')))
	local code = [[foo bar]]
	local tags = {IDENTIFIER, 'foo', IDENTIFIER, 'bar'}
	assert_lex(lex, code, tags)

	-- Modify the identifier rule to not catch keywords.
	lex:modify_rule('identifier', -lpeg.P('foo') * lex:tag(IDENTIFIER, lexer.word))
	tags = {KEYWORD, 'foo', IDENTIFIER, 'bar'}
	assert_lex(lex, code, tags)
end

-- Tests a basic lexer with a couple of simple rules and a custom tag.
function test_add_tag()
	local lex = lexer.new('test')
	assert_default_tags(lex)
	lex:add_rule('keyword', lex:tag('custom', word_match('foo bar baz')))
	assert_default_tags(lex)
	local code = [[foo bar baz]]
	local tags = {'custom', 'foo', 'custom', 'bar', 'custom', 'baz'}
	assert_lex(lex, code, tags)
end

-- Tests word lists.
function test_word_list()
	local lex = lexer.new('test')
	lex:add_rule('keyword', lex:tag(KEYWORD, lex:word_match(KEYWORD)))
	lex:add_rule('identifier', lex:tag(IDENTIFIER, lexer.word))
	lex:add_rule('operator', lex:tag(OPERATOR, '.'))
	local code = [[foo bar.baz quux]]
	local tags = {
		IDENTIFIER, 'foo', --
		IDENTIFIER, 'bar', --
		OPERATOR, '.', --
		IDENTIFIER, 'baz', --
		IDENTIFIER, 'quux'
	}
	assert_lex(lex, code, tags)

	lex:set_word_list(KEYWORD, 'foo quux')
	tags[1 * 2 - 1], tags[1 * 2] = KEYWORD, 'foo'
	tags[5 * 2 - 1], tags[5 * 2] = KEYWORD, 'quux'
	assert_lex(lex, code, tags)

	lex:set_word_list(KEYWORD, 'bar', true) -- append
	tags[2 * 2 - 1], tags[2 * 2] = KEYWORD, 'bar'
	assert_lex(lex, code, tags)

	lex:set_word_list(KEYWORD, {'bar.baz'})
	tags = {IDENTIFIER, 'foo', KEYWORD, 'bar.baz', IDENTIFIER, 'quux'}
	assert_lex(lex, code, tags)
end

-- Tests a simple parent lexer embedding a simple child lexer.
-- Ensures the child's custom tags are also copied over.
function test_embed()
	-- Create the parent lexer.
	local parent = lexer.new('parent')
	assert_default_tags(parent)
	parent:add_rule('identifier', parent:tag('parent', lexer.word))

	-- Create the child lexer.
	local child = lexer.new('child')
	assert_default_tags(child)
	child:add_rule('number', child:tag('child', lexer.integer))

	-- Assert the child's tags are not embedded in the parent yet.
	assert(not parent._TAGS['whitespace.' .. child._name])
	assert(not parent._extra_tags['whitespace.' .. child._name])
	assert(not parent._TAGS['child'])
	assert(not parent._extra_tags['child'])

	-- Embed the child into the parent and verify the child's tags were copied over.
	local start_rule = parent:tag('transition', lpeg.P('['))
	local end_rule = parent:tag('transition', lpeg.P(']'))
	parent:embed(child, start_rule, end_rule)
	assert_default_tags(parent)

	-- Lex some parent -> child -> parent code.
	local code = [[foo [1, 2, 3] bar]]
	local tags = {
		'parent', 'foo', --
		'transition', '[', --
		'child', '1', DEFAULT, ',', 'child', '2', DEFAULT, ',', 'child', '3', --
		'transition', ']', --
		'parent', 'bar'
	}
	assert_lex(parent, code, tags)

	-- Lex some child -> parent code, starting from within the child.
	code = [[2, 3] bar]]
	tags = {
		'child', '2', DEFAULT, ',', 'child', '3', 'transition', ']', --
		'parent', 'bar'
	}
	local initial_style = parent._TAGS['whitespace.' .. child._name]
	assert_lex(parent, code, tags, initial_style)
end

-- Tests a simple child lexer embedding itself within a simple parent lexer.
-- Ensures the child's custom tags are also copied over.
function test_embed_into()
	-- Create the child lexer.
	local child = lexer.new('child')
	child:add_rule('number', child:tag('child', lexer.integer))

	-- Create the parent lexer.
	local parent = lexer.new('parent')
	parent:add_rule('identifier', parent:tag('parent', lexer.word))

	-- Embed the child within the parent and verify the child's custom tags were copied over.
	local start_rule = parent:tag('transition', lpeg.P('['))
	local end_rule = parent:tag('transition', lpeg.P(']'))
	parent:embed(child, start_rule, end_rule)
	assert_default_tags(parent)

	-- Verify any subsequent fold point additions to the child are copied to the parent.
	child:add_fold_point('transition', '[', ']')
	assert(parent._fold_points['transition']['['] == 1)
	assert(parent._fold_points['transition'][']'] == -1)

	-- Lex some parent -> child -> parent code.
	local code = [[foo [1, 2, 3] bar]]
	local tags = {
		'parent', 'foo', --
		'transition', '[', --
		'child', '1', DEFAULT, ',', 'child', '2', DEFAULT, ',', 'child', '3', --
		'transition', ']', --
		'parent', 'bar'
	}
	assert_lex(child, code, tags)

	-- Lex some child -> parent code, starting from within the child.
	code = [[2, 3] bar]]
	tags = {
		'child', '2', DEFAULT, ',', 'child', '3', 'transition', ']', --
		'parent', 'bar'
	}
	local initial_style = parent._TAGS['whitespace.' .. child._name]
	assert_lex(child, code, tags, initial_style)

	-- Fold some code.
	code = [[
    foo [
      1, 2, 3
    ] bar
    baz
  ]]
	local folds = {1, -3}
	local levels = assert_fold_points(child, code, folds)
	assert(levels[3] > levels[4]) -- verify ']' is fold end point
end

-- Tests a proxy lexer that inherits from a simple parent lexer and embeds a simple child lexer.
-- Ensures both the proxy's and child's custom tags are also copied over.
function test_proxy()
	-- Create the parent lexer.
	local parent = lexer.new('parent')
	parent:add_rule('identifier', parent:tag('parent', lexer.word))

	-- Create the child lexer.
	local child = lexer.new('child')
	child:add_rule('number', child:tag('child', lexer.integer))

	-- Create the proxy lexer.
	local proxy = lexer.new('proxy', {inherit = parent})

	-- Embed the child into the parent and verify the proxy's custom tag was copied over.
	local start_rule = proxy:tag('transition', lpeg.P('['))
	local end_rule = proxy:tag('transition', lpeg.P(']'))
	proxy:embed(child, start_rule, end_rule)

	-- Lex some parent -> child -> parent code.
	local code = [[foo [1, 2, 3] bar]]
	local tags = {
		'parent', 'foo', --
		'transition', '[', --
		'child', '1', DEFAULT, ',', 'child', '2', DEFAULT, ',', 'child', '3', --
		'transition', ']', --
		'parent', 'bar'
	}
	assert_lex(proxy, code, tags)

	-- Lex some child -> parent code, starting from within the child.
	code = [[ 2, 3] bar]]
	tags = {
		'child', '2', DEFAULT, ',', 'child', '3', 'transition', ']', --
		'parent', 'bar'
	}
	local initial_style = parent._TAGS['whitespace.' .. child._name]
	assert_lex(proxy, code, tags, initial_style)

	-- Verify any subsequent fold point additions to the proxy are copied to the parent.
	proxy:add_fold_point('transition', '[', ']')
	assert(parent._fold_points['transition']['['] == 1)
	assert(parent._fold_points['transition'][']'] == -1)

	-- Fold some code.
	code = [[
    foo [
      1, 2, 3
    ] bar
    baz
  ]]
	local folds = {1, -3}
	local levels = assert_fold_points(proxy, code, folds)
	assert(levels[3] > levels[4]) -- verify ']' is fold end point
end

-- Tests a lexer that inherits from another one.
function test_inherits_rules()
	local lex = lexer.new('test')
	lex:add_rule('keyword', lex:tag(KEYWORD, word_match('foo bar baz')))

	-- Verify inherited rules are used.
	local sublexer = lexer.new('test2', {inherit = lex})
	local code = [[foo bar baz]]
	local tags = {KEYWORD, 'foo', KEYWORD, 'bar', KEYWORD, 'baz'}
	assert_lex(sublexer, code, tags)

	-- Verify subsequently added rules are also used.
	sublexer:add_rule('keyword2', sublexer:tag(KEYWORD, lpeg.P('quux')))
	code = [[foo bar baz quux]]
	tags = {KEYWORD, 'foo', KEYWORD, 'bar', KEYWORD, 'baz', KEYWORD, 'quux'}
	assert_lex(sublexer, code, tags)
end

-- Tests that fold words are folded properly, even if fold words are substrings of others
-- (e.g. "if" and "endif").
function test_fold_words()
	local lex = lexer.new('test')
	lex:add_rule('keyword', lex:tag(KEYWORD, word_match('if endif')))
	lex:add_fold_point(KEYWORD, 'if', 'endif')

	local code = [[
    if foo
      bar
    endif
    ifbaz
    quuxif
  ]]
	local folds = {1, -3}
	local levels = assert_fold_points(lex, code, folds)
	assert(levels[2] == lexer.FOLD_BASE + 1)
	assert(levels[3] == lexer.FOLD_BASE + 1)
	assert(levels[4] == lexer.FOLD_BASE)
end

-- Tests folding by indentation.
function test_fold_by_indentation()
	local lex = lexer.new('test', {fold_by_indentation = true})
	local code = [[
    if foo:
      bar
    else:
      baz
  ]]
	lexer.fold_level = {lexer.FOLD_BASE} -- Scintilla normally creates this
	local folds = {1, -2, 3}
	assert_fold_points(lex, code, folds)
end

-- Tests that all lexers load and lex text.
function test_loads()
	local p = io.popen('ls -1 lexers/*.lua')
	local files = p:read('*a')
	p:close()
	for file in files:gmatch('[^\n]+') do
		local lex_name = file:match('^lexers/(.+)%.lua$')
		if lex_name ~= 'lexer' then
			local lex = lexer.load(lex_name, nil, true)
			assert_default_tags(lex)
			local tags = lex:lex('test')
			assert(#tags >= 2)
		end
	end
end

function test_names()
	lexer.property['scintillua.lexers'] = ''
	local names = lexer.names()
	assert(#names > 0)
	local lua_found = false
	for _, name in ipairs(names) do
		assert(name ~= 'lexer')
		if name == 'lua' then lua_found = true end
	end
	assert(lua_found)
	local names2 = lexer.names('lexers')
	assert(#names == #names2)
	lexer.property['scintillua.lexers'] = 'lexers'
	local names3 = lexer.names()
	assert(#names == #names3)
end

-- Tests the Lua lexer.
function test_lua()
	local lua = lexer.load('lua')
	assert(lua._name == 'lua')
	assert_default_tags(lua)
	local rules = {
		'keyword', 'function', 'constant', 'identifier', 'string', 'comment', 'number', 'label',
		'attribute', 'operator'
	}
	assert_rules(lua, rules)
	local tags = {
		STRING .. '.longstring', --
		'whitespace.lua' -- language-specific whitespace for multilang lexers
	}
	assert_extra_tags(lua, tags)

	-- Lexing tests.
	local code = [=[
    --[[ Comment. ]]--
    ::begin::
    local a <const> = -1 + 2.0e3 - 0x40
    local b = "two"..[[three]] .. 'four\''
    c ={_G.print, type = foo{math.pi}}
    print(string.upper'a', b:upper())
  ]=]
	tags = {
		COMMENT, '--[[ Comment. ]]', COMMENT, '--', --
		LABEL, '::begin::', --
		KEYWORD, 'local', IDENTIFIER, 'a', ATTRIBUTE, '<const>', --
		OPERATOR, '=', --
		NUMBER, '-1', OPERATOR, '+', NUMBER, '2.0e3', OPERATOR, '-', NUMBER, '0x40', --
		KEYWORD, 'local', IDENTIFIER, 'b', --
		OPERATOR, '=', --
		STRING, '"two"', --
		OPERATOR, '..', --
		STRING .. '.longstring', '[[three]]', --
		OPERATOR, '..', --
		STRING, "'four\\''", --
		IDENTIFIER, 'c', --
		OPERATOR, '=', --
		OPERATOR, '{', --
		CONSTANT_BUILTIN, '_G', OPERATOR, '.', IDENTIFIER, 'print', --
		OPERATOR, ',', --
		IDENTIFIER, 'type', --
		OPERATOR, '=', --
		FUNCTION, 'foo', OPERATOR, '{', CONSTANT_BUILTIN, 'math.pi', OPERATOR, '}', OPERATOR, '}', --
		FUNCTION_BUILTIN, 'print', --
		OPERATOR, '(', --
		FUNCTION_BUILTIN, 'string.upper', STRING, "'a'", --
		OPERATOR, ',', --
		IDENTIFIER, 'b', OPERATOR, ':', FUNCTION_METHOD, 'upper', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, ')'
	}
	assert_lex(lua, code, tags)

	-- Folding tests.
	code = [=[
    if foo then
      bar
    end
    for k, v in pairs(foo) do
      bar
    end
    function foo(bar)
      baz
    end
    repeat
      foo
    until bar
    --[[
      foo
    ]]
    (foo,
     bar,
     baz)
    {foo,
     bar,
     baz}
  ]=]
	local folds = {1, -3, 4, -6, 7, -9, 10, -12, 13, -15, 16, -18, 19}
	assert_fold_points(lua, code, folds)

	-- Test overriding keywords.
	lua:set_word_list(KEYWORD, '')
	assert_lex(lua, 'if', {IDENTIFIER, 'if'})

	-- Test adding to built-in functions.
	lua:set_word_list(FUNCTION_BUILTIN, 'module', true) -- from Lua 5.1
	assert_lex(lua, 'dofile(', {FUNCTION_BUILTIN, 'dofile', OPERATOR, '('})
	assert_lex(lua, 'module(', {FUNCTION_BUILTIN, 'module', OPERATOR, '('})
end

-- Tests the C lexer.
function test_c()
	local c = lexer.load('c')
	assert(c._name == 'c')
	assert_default_tags(c)

	-- Lexing tests.
	local code = ([[
    /* Comment. */
    #include <stdio.h>
    #include "lua.h"
    #  define INT_MAX_ 1
    int main(int argc, char **argv) {
    begin:
      if (NULL) // comment
        printf("%ld %f %s %i", 1l, 1.0e-1f, L"foo", INT_MAX);
      foo.free, foo->free(), free(foo);
      return 0x0?argc:0;
    }
  ]]):gsub('    ', '') -- strip indent
	local tags = {
		COMMENT, '/* Comment. */', --
		PREPROCESSOR, '#include', STRING, '<stdio.h>', --
		PREPROCESSOR, '#include', STRING, '"lua.h"', --
		PREPROCESSOR, '#  define', IDENTIFIER, 'INT_MAX_', NUMBER, '1', --
		TYPE, 'int', FUNCTION, 'main', --
		OPERATOR, '(', --
		TYPE, 'int', IDENTIFIER, 'argc', --
		OPERATOR, ',', --
		TYPE, 'char', OPERATOR, '*', OPERATOR, '*', IDENTIFIER, 'argv', --
		OPERATOR, ')', --
		OPERATOR, '{', --
		LABEL, 'begin:', --
		KEYWORD, 'if', OPERATOR, '(', CONSTANT_BUILTIN, 'NULL', OPERATOR, ')', COMMENT, '// comment', --
		FUNCTION_BUILTIN, 'printf', --
		OPERATOR, '(', --
		STRING, '"%ld %f %s %i"', --
		OPERATOR, ',', --
		NUMBER, '1l', --
		OPERATOR, ',', --
		NUMBER, '1.0e-1f', --
		OPERATOR, ',', --
		STRING, 'L"foo"', --
		OPERATOR, ',', --
		CONSTANT_BUILTIN, 'INT_MAX', --
		OPERATOR, ')', --
		OPERATOR, ';', --
		IDENTIFIER, 'foo', OPERATOR, '.', IDENTIFIER, 'free', --
		OPERATOR, ',', --
		IDENTIFIER, 'foo', --
		OPERATOR, '-', OPERATOR, '>', --
		FUNCTION_METHOD, 'free', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, ',', --
		FUNCTION_BUILTIN, 'free', OPERATOR, '(', IDENTIFIER, 'foo', OPERATOR, ')', --
		OPERATOR, ';', --
		KEYWORD, 'return', --
		NUMBER, '0x0', --
		OPERATOR, '?', --
		IDENTIFIER, 'argc', -- should not be a label
		OPERATOR, ':', --
		NUMBER, '0', --
		OPERATOR, ';', --
		OPERATOR, '}'
	}
	assert_lex(c, code, tags)

	-- Folding tests.
	code = ([[
    if (foo) {
      bar;
    }
    /**
     * foo
     */
    #ifdef foo
      bar;
    #endif
  ]]):gsub('    ', '') -- strip indent
	local folds = {1, -3, 4, -6, 7}
	assert_fold_points(c, code, folds)
end

-- Tests the HTML lexer and its embedded languages.
function test_html()
	local html = lexer.load('html')
	assert(html._name == 'html')
	assert_default_tags(html)
	local rules = {
		'comment', 'doctype', 'tag', 'tag_close', 'attribute', -- 'equals',
		'string', 'number', 'entity'
	}
	assert_rules(html, rules)
	local tags = {
		TAG .. '.doctype', TAG .. '.unknown', ATTRIBUTE .. '.unknown', --
		'whitespace.html', -- HTML
		'property', 'whitespace.css', -- CSS
		'whitespace.javascript', -- JS
		'whitespace.coffeescript' -- CoffeeScript
	}
	assert_extra_tags(html, tags)
	assert_children(html, {'css', 'css.style', 'javascript', 'coffeescript'})

	-- Lexing tests.
	local code = [[
    <!DOCTYPE html>
    <!-- Comment. -->
    <html>
      <HEAD>
        <style type="text/css">
          @charset "utf8"
          /* Another comment. */
          h1:hover, h2::first-line {
            color: red;
            border: 1.5px solid #0000FF;
            background: url("/images/image.jpg");
          }
          table.class {}
        </style>
        <script type="text/javascript">
          /* A third comment. */
          var a = 1 + 2.0e3 - 0x40;
          var b = "two" + `three`;
          var c = /pattern/i;
          foo(eval(arguments), bar.baz(), Object);
        </script>
      </HEAD>
      <bod clss = "unknown">
      <hr tabindex=1/> &copy;
      <div style="float: right">
    </html>
  ]]
	local tag_chars = TAG .. '.chars'
	tags = {
		TAG .. '.doctype', '<!DOCTYPE html>', --
		COMMENT, '<!-- Comment. -->', --
		tag_chars, '<', TAG, 'html', tag_chars, '>', --
		tag_chars, '<', TAG, 'HEAD', tag_chars, '>', --
		tag_chars, '<', TAG, 'style', --
		ATTRIBUTE, 'type', OPERATOR, '=', STRING, '"text/css"', --
		tag_chars, '>', --
		PREPROCESSOR, '@charset', STRING, '"utf8"', --
		COMMENT, '/* Another comment. */', --
		TAG, 'h1', 'pseudoclass', ':hover', --
		OPERATOR, ',', --
		TAG, 'h2', 'pseudoelement', '::first-line', --
		OPERATOR, '{', --
		'property', 'color', OPERATOR, ':', CONSTANT_BUILTIN, 'red', OPERATOR, ';', --
		'property', 'border', --
		OPERATOR, ':', --
		NUMBER, '1.5px', CONSTANT_BUILTIN, 'solid', NUMBER, '#0000FF', --
		OPERATOR, ';', --
		'property', 'background', --
		OPERATOR, ':', --
		FUNCTION_BUILTIN, 'url', OPERATOR, '(', STRING, '"/images/image.jpg"', OPERATOR, ')', --
		OPERATOR, ';', --
		OPERATOR, '}', --
		TAG, 'table', OPERATOR, '.', IDENTIFIER, 'class', OPERATOR, '{', OPERATOR, '}', --
		tag_chars, '</', TAG, 'style', tag_chars, '>', --
		tag_chars, '<', TAG, 'script', --
		ATTRIBUTE, 'type', OPERATOR, '=', STRING, '"text/javascript"', --
		tag_chars, '>', --
		COMMENT, '/* A third comment. */', --
		KEYWORD, 'var', IDENTIFIER, 'a', --
		OPERATOR, '=', --
		NUMBER, '1', OPERATOR, '+', NUMBER, '2.0e3', OPERATOR, '-', NUMBER, '0x40', --
		OPERATOR, ';', --
		KEYWORD, 'var', IDENTIFIER, 'b', --
		OPERATOR, '=', --
		STRING, '"two"', OPERATOR, '+', STRING, '`three`', --
		OPERATOR, ';', --
		KEYWORD, 'var', IDENTIFIER, 'c', OPERATOR, '=', REGEX, '/pattern/i', OPERATOR, ';', --
		FUNCTION, 'foo', --
		OPERATOR, '(', --
		FUNCTION_BUILTIN, 'eval', OPERATOR, '(', CONSTANT_BUILTIN, 'arguments', OPERATOR, ')', --
		OPERATOR, ',', --
		IDENTIFIER, 'bar', OPERATOR, '.', FUNCTION_METHOD, 'baz', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, ',', --
		TYPE, 'Object', --
		OPERATOR, ')', --
		OPERATOR, ';', --
		tag_chars, '</', TAG, 'script', tag_chars, '>', --
		tag_chars, '</', TAG, 'HEAD', tag_chars, '>', --
		tag_chars, '<', TAG .. '.unknown', 'bod', --
		ATTRIBUTE .. '.unknown', 'clss', OPERATOR, '=', STRING, '"unknown"', --
		tag_chars, '>', --
		tag_chars, '<', TAG .. '.single', 'hr', --
		ATTRIBUTE, 'tabindex', OPERATOR, '=', NUMBER, '1', --
		tag_chars, '/>', --
		CONSTANT_BUILTIN .. '.entity', '&copy;', --
		tag_chars, '<', TAG, 'div', --
		ATTRIBUTE, 'style', OPERATOR, '=', STRING, '"', --
		'property', 'float', OPERATOR, ':', CONSTANT_BUILTIN, 'right', --
		STRING, '"', tag_chars, '>', --
		tag_chars, '</', TAG, 'html', tag_chars, '>'
	}
	assert_lex(html, code, tags)

	-- Folding tests.
	local symbols = {'<', '<!--', '-->', '{', '}', '/*', '*/'}
	for i = 1, #symbols do assert(html._fold_points._symbols[symbols[i]]) end
	assert(html._fold_points[TAG .. '.chars']['<'])
	assert(html._fold_points[COMMENT]['<!--'])
	assert(html._fold_points[COMMENT]['-->'])
	assert(html._fold_points[OPERATOR]['{'])
	assert(html._fold_points[OPERATOR]['}'])
	assert(html._fold_points[COMMENT]['/*'])
	assert(html._fold_points[COMMENT]['*/'])
	code = [[
    <html>
      foo
    </html>
    <body/>
    <style type="text/css">
      h1 {
        foo;
      }
    </style>
    <script type="text/javascript">
      function foo() {
        bar;
      }
    </script>
    h1 {
      foo;
    }
    function foo() {
      bar;
    }
  ]]
	local folds = {1, -3, 5, 6, -8, -9, 10, 11, -13}
	local levels = assert_fold_points(html, code, folds)
	assert(levels[3] > levels[4]) -- </html> is ending fold point
end

-- Tests the PHP lexer.
function test_php()
	local php = lexer.load('php')
	assert(php._name == 'php')
	assert_default_tags(php)
	assert_extra_tags(php, {'whitespace.php'})

	-- Lexing tests
	-- Starting in HTML.
	local code = [[<h1><?php echo "hi" . PHP_OS . foo() . bar->baz(); ?></h1>]]
	local tag_chars = TAG .. '.chars'
	local tags = {
		tag_chars, '<', TAG, 'h1', tag_chars, '>', --
		PREPROCESSOR, '<?php ', --
		KEYWORD, 'echo', --
		STRING, '"hi"', --
		OPERATOR, '.', --
		CONSTANT_BUILTIN, 'PHP_OS', --
		OPERATOR, '.', --
		FUNCTION, 'foo', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, '.', --
		IDENTIFIER, 'bar', --
		OPERATOR, '-', OPERATOR, '>', --
		FUNCTION_METHOD, 'baz', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, ';', --
		PREPROCESSOR, '?>', --
		tag_chars, '</', TAG, 'h1', tag_chars, '>'
	}
	local initial_style = php._TAGS['whitespace.html']
	assert_lex(php, code, tags, initial_style)
	initial_style = php._TAGS['default'] -- also test non-ws init style
	assert_lex(php, code, tags, initial_style)
	initial_style = php._TAGS['default'] -- also test non-ws init style
	assert_lex(php, code, tags, initial_style)
	-- Starting in PHP.
	code = [[echo "hi";]]
	initial_style = php._TAGS['whitespace.php']
	tags = {KEYWORD, 'echo', STRING, '"hi"', OPERATOR, ';'}
	assert_lex(php, code, tags, initial_style)

	-- Folding tests.
	local symbols = {'<?', '?>', '/*', '*/', '{', '}', '(', ')'}
	for i = 1, #symbols do assert(php._fold_points._symbols[symbols[i]]) end
	assert(php._fold_points[PREPROCESSOR]['<?'])
	assert(php._fold_points[PREPROCESSOR]['?>'])
	assert(php._fold_points[COMMENT]['/*'])
	assert(php._fold_points[COMMENT]['*/'])
	assert(php._fold_points[OPERATOR]['{'])
	assert(php._fold_points[OPERATOR]['}'])
	assert(php._fold_points[OPERATOR]['('])
	assert(php._fold_points[OPERATOR][')'])
end

-- Tests the Ruby lexer.
function test_ruby()
	local ruby = lexer.load('ruby')

	-- Lexing tests.
	local code = [[
    # Comment.
    require "foo"
    $a = 1 + 2.0e3 - 0x4_0 if true
    b = "two" + %q[three] + <<-FOUR
      four
    FOUR
    puts :c, foo.puts
  ]]
	local tags = {
		COMMENT, '# Comment.', --
		FUNCTION_BUILTIN, 'require', STRING, '"foo"', --
		VARIABLE, '$a', --
		OPERATOR, '=', --
		NUMBER, '1', OPERATOR, '+', NUMBER, '2.0e3', OPERATOR, '-', NUMBER, '0x4_0', --
		KEYWORD, 'if', KEYWORD, 'true', --
		IDENTIFIER, 'b', --
		OPERATOR, '=', --
		STRING, '"two"', --
		OPERATOR, '+', --
		STRING, '%q[three]', --
		OPERATOR, '+', --
		STRING, '<<-FOUR\n      four\n    FOUR', --
		FUNCTION_BUILTIN, 'puts', STRING .. '.symbol', ':c', --
		OPERATOR, ',', --
		IDENTIFIER, 'foo', OPERATOR, '.', IDENTIFIER, 'puts'
	}
	assert_lex(ruby, code, tags)

	-- Folding tests.
	local fold_keywords = {
		begin = 1, class = 1, def = 1, ['do'] = 1, ['for'] = 1, module = 1, case = 1,
		['if'] = function() end, ['while'] = function() end, unless = function() end,
		['until'] = function() end, ['end'] = -1
	}
	for k, v in pairs(fold_keywords) do
		assert(ruby._fold_points._symbols[k])
		if type(v) == 'number' then
			assert(ruby._fold_points[KEYWORD][k] == v)
		else
			assert(type(ruby._fold_points[KEYWORD][k]) == 'function')
		end
	end
	local fold_operators = {'(', ')', '[', ']', '{', '}'}
	for i = 1, #fold_operators do
		assert(ruby._fold_points._symbols[fold_operators[i]])
		assert(ruby._fold_points[OPERATOR][fold_operators[i]])
	end
	code = [=[
    class Foo
      bar
    end
    foo.each do |v|
      bar
    end
    def foo(bar)
      baz
    end
    =begin
      foo
    =end
    (foo,
     bar,
     baz)
    [foo,
     bar,
     baz]
    {foo,
     bar,
     baz}
  ]=]
	local folds = {1, -3, 4, -6, 7, -9, 10, -12, 13, -15, 16, -18, 19, -21}
	assert_fold_points(ruby, code, folds)
end

-- Tests the Ruby and Rails lexers.
-- The Rails lexer inherits from Ruby and modifies some of its rules. Verify the Ruby lexer
-- is unaffected.
function test_ruby_and_rails()
	local ruby = lexer.load('ruby', nil, true)
	local rails = lexer.load('rails', nil, true)
	local code = [[
    class Foo < ActiveRecord::Base
      has_one :bar
    end
  ]]
	local ruby_tags = {
		KEYWORD, 'class', IDENTIFIER, 'Foo', --
		OPERATOR, '<', --
		IDENTIFIER, 'ActiveRecord', OPERATOR, ':', OPERATOR, ':', IDENTIFIER, 'Base', --
		IDENTIFIER, 'has_one', -- function.builtin in rails
		STRING .. '.symbol', ':bar', --
		KEYWORD, 'end'
	}
	assert_lex(ruby, code, ruby_tags)

	local rails_tags = {
		KEYWORD, 'class', IDENTIFIER, 'Foo', --
		OPERATOR, '<', --
		IDENTIFIER, 'ActiveRecord', OPERATOR, ':', OPERATOR, ':', IDENTIFIER, 'Base', --
		FUNCTION_BUILTIN, 'has_one', STRING .. '.symbol', ':bar', --
		KEYWORD, 'end'
	}
	assert_lex(rails, code, rails_tags)
end

-- Tests the RHTML lexer, which is a proxy for HTML and Rails.
function test_rhtml()
	local rhtml = lexer.load('rhtml')

	-- Lexing tests.
	-- Start in HTML.
	local code = [[<h1><% puts "hi" + link_to "foo" @foo %></h1>]]
	local tag_chars = TAG .. '.chars'
	local rhtml_tags = {
		tag_chars, '<', TAG, 'h1', tag_chars, '>', --
		lexer.PREPROCESSOR, '<%', --
		FUNCTION_BUILTIN, 'puts', STRING, '"hi"', --
		OPERATOR, '+', --
		FUNCTION_BUILTIN, 'link_to', STRING, '"foo"', VARIABLE, '@foo', --
		lexer.PREPROCESSOR, '%>', --
		tag_chars, '</', TAG, 'h1', tag_chars, '>'
	}
	local initial_style = rhtml._TAGS['whitespace.html']
	assert_lex(rhtml, code, rhtml_tags, initial_style)
	-- Start in Ruby.
	code = [[puts "hi" + link_to "foo" @foo]]
	rhtml_tags = {
		FUNCTION_BUILTIN, 'puts', STRING, '"hi"', --
		OPERATOR, '+', --
		FUNCTION_BUILTIN, 'link_to', STRING, '"foo"', VARIABLE, '@foo'
	}
	initial_style = rhtml._TAGS['whitespace.rails']
	assert_lex(rhtml, code, rhtml_tags, initial_style)
end

-- Tests folding with complex keywords and case-insensitivity.
function test_vb_folding()
	local vb = lexer.load('vb')
	local code = [[
    Sub Foo()
      If bar Then

      End If
    End Sub

    sub baz()

    end sub
  ]]
	local folds = {1, 2, -4, -5, 7, -9}
	assert_fold_points(vb, code, folds)
end

-- Tests the Makefile lexer with bash/shell embedded two different ways.
function test_makefile()
	local makefile = lexer.load('makefile')
	local code = ([[
    # Comment.
    .DEFAULT_GOAL := all
    foo ?= bar=baz
    all: $(foo)
    $(foo): ; echo 'hi'

    .PHONY: docs
    define build-cc =
      $(CC) ${CFLAGS} -c $< -o $@
    endef
     func = $(call quux, ${adsuffix .o, $(1)})
     echo = $(shell echo $PATH)
  ]]):gsub('    ', ''):gsub('  ', '\t') -- strip indent, convert to tabs
	local tags = {
		COMMENT, '# Comment.', --
		VARIABLE_BUILTIN, '.DEFAULT_GOAL', OPERATOR, ':=', IDENTIFIER, 'all', --
		VARIABLE, 'foo', OPERATOR, '?=', IDENTIFIER, 'bar', DEFAULT, '=', IDENTIFIER, 'baz', --
		IDENTIFIER, 'all', OPERATOR, ':', OPERATOR, '$(', VARIABLE, 'foo', OPERATOR, ')', --
		OPERATOR, '$(', VARIABLE, 'foo', OPERATOR, ')', --
		OPERATOR, ':', OPERATOR, ';', --
		FUNCTION_BUILTIN, 'echo', STRING, "'hi'", --
		CONSTANT_BUILTIN, '.PHONY', OPERATOR, ':', IDENTIFIER, 'docs', --
		KEYWORD, 'define', FUNCTION, 'build-cc', OPERATOR, '=', --
		OPERATOR, '$(', VARIABLE_BUILTIN, 'CC', OPERATOR, ')', --
		OPERATOR, '${', VARIABLE_BUILTIN, 'CFLAGS', OPERATOR, '}', --
		DEFAULT, '-c', --
		OPERATOR, '$', VARIABLE_BUILTIN, '<', --
		DEFAULT, '-o', --
		OPERATOR, '$', VARIABLE_BUILTIN, '@', --
		KEYWORD, 'endef', --
		FUNCTION, 'func', --
		OPERATOR, '=', --
		OPERATOR, '$(', FUNCTION_BUILTIN, 'call', --
		FUNCTION, 'quux', --
		DEFAULT, ',', --
		OPERATOR, '${', --
		VARIABLE, 'adsuffix', -- typo should not be tagged as FUNCTION_BUILTIN
		IDENTIFIER, '.o', --
		DEFAULT, ',', --
		OPERATOR, '$(', VARIABLE, '1', OPERATOR, ')', --
		OPERATOR, '}', --
		OPERATOR, ')', --
		VARIABLE, 'echo', --
		OPERATOR, '=', --
		OPERATOR, '$(', FUNCTION_BUILTIN, 'shell', --
		FUNCTION_BUILTIN, 'echo', OPERATOR, '$', VARIABLE_BUILTIN, 'PATH', --
		OPERATOR, ')'
	}
	assert_lex(makefile, code, tags)
end

-- Tests the Bash lexer, where some operators are recognized only in certain contexts.
function test_bash()
	local bash = lexer.load('bash')

	-- Lexing tests.
	local code = [=[
    # Comment.
    foo=bar=baz:$PATH
    echo -n $foo 1>&2
    if [ ! -z "foo" -a 0 -ne 1 ]; then
      quux=$((1 - 2 / 0x3))
    elif [[ -d /foo/bar-baz.quux ]]; then
      foo=${?}
    fi
    s=<<-"END"
      foobar
    END
  ]=]
	local tags = {
		COMMENT, '# Comment.', --
		VARIABLE, 'foo', --
		OPERATOR, '=', --
		IDENTIFIER, 'bar', DEFAULT, '=', IDENTIFIER, 'baz', --
		DEFAULT, ':', --
		OPERATOR, '$', VARIABLE_BUILTIN, 'PATH', --
		FUNCTION_BUILTIN, 'echo', --
		DEFAULT, '-n', --
		OPERATOR, '$', VARIABLE, 'foo', --
		NUMBER, '1', OPERATOR, '>', OPERATOR, '&', NUMBER, '2', --
		KEYWORD, 'if', --
		OPERATOR, '[', --
		OPERATOR, '!', DEFAULT, '-z', STRING, '"foo"', --
		DEFAULT, '-a', --
		NUMBER, '0', DEFAULT, '-ne', NUMBER, '1', --
		OPERATOR, ']', --
		OPERATOR, ';', KEYWORD, 'then', --
		VARIABLE, 'quux', --
		OPERATOR, '=', --
		OPERATOR, '$', OPERATOR, '(', OPERATOR, '(', --
		NUMBER, '1', DEFAULT, '-', NUMBER, '2', DEFAULT, '/', NUMBER, '0x3', --
		OPERATOR, ')', OPERATOR, ')', --
		KEYWORD, 'elif', --
		OPERATOR, '[', OPERATOR, '[', --
		DEFAULT, '-d', --
		DEFAULT, '/', --
		IDENTIFIER, 'foo', --
		DEFAULT, '/', --
		IDENTIFIER, 'bar', DEFAULT, '-baz', DEFAULT, '.', IDENTIFIER, 'quux', --
		OPERATOR, ']', OPERATOR, ']', OPERATOR, ';', KEYWORD, 'then', --
		VARIABLE, 'foo', OPERATOR, '=', OPERATOR, '${', VARIABLE_BUILTIN, '?', OPERATOR, '}', --
		KEYWORD, 'fi', --
		VARIABLE, 's', OPERATOR, '=', STRING, '<<-"END"\n      foobar\n    END'
	}
	assert_lex(bash, code, tags)

	-- Folding tests.
	code = [[
    function foo() {

    }
    if [ expr ]; then
      case
        *)
        ;;
      esac
    fi
    for x in y; do

    done
  ]]
	local folds = {1, -3, 4, 5, -8, -9, 10, -12}
	assert_fold_points(bash, code, folds)
end

-- Tests the C++ lexer.
function test_cpp()
	local cpp = lexer.load('cpp')
	local code = [=[
    /*/*Comment.*///
    #include <string>
    #include "header.h"
    #  undef FOO
    [[deprecated]]
    class Foo : public Bar
    {
      Foo();
      ~Foo();
    private:
      std::string mFoo = u8"foo";
      int mBar = 1;
    };

    Foo::Foo()
    {
      std::clog << std::abs(strlen(mFoo.c_str()));
      this->bar(1'000 + 0xFF'00 - 0b11'00);
      std::sort(
    }
  ]=]
	local tags = {
		COMMENT, '/*/*Comment.*/', COMMENT, '//', --
		PREPROCESSOR, '#include', STRING, '<string>', --
		PREPROCESSOR, '#include', STRING, '"header.h"', --
		PREPROCESSOR, '#  undef', IDENTIFIER, 'FOO', --
		ATTRIBUTE, '[[deprecated]]', --
		KEYWORD, 'class', IDENTIFIER, 'Foo', OPERATOR, ':', KEYWORD, 'public', IDENTIFIER, 'Bar', --
		OPERATOR, '{', --
		FUNCTION, 'Foo', OPERATOR, '(', OPERATOR, ')', OPERATOR, ';', --
		OPERATOR, '~', FUNCTION, 'Foo', OPERATOR, '(', OPERATOR, ')', OPERATOR, ';', --
		KEYWORD, 'private', OPERATOR, ':', --
		TYPE .. '.stl', 'std::string', --
		IDENTIFIER, 'mFoo', --
		OPERATOR, '=', --
		STRING, 'u8"foo"', --
		OPERATOR, ';', --
		TYPE, 'int', IDENTIFIER, 'mBar', OPERATOR, '=', NUMBER, '1', OPERATOR, ';', --
		OPERATOR, '}', OPERATOR, ';', --
		IDENTIFIER, 'Foo', OPERATOR, ':', OPERATOR, ':', FUNCTION, 'Foo', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, '{', --
		CONSTANT_BUILTIN .. '.stl', 'std::clog', --
		OPERATOR, '<', OPERATOR, '<', --
		FUNCTION_BUILTIN, 'std::abs', --
		OPERATOR, '(', --
		FUNCTION_BUILTIN, 'strlen', --
		OPERATOR, '(', --
		IDENTIFIER, 'mFoo', OPERATOR, '.', FUNCTION_METHOD, 'c_str', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, ')', --
		OPERATOR, ')', --
		OPERATOR, ';', --
		KEYWORD, 'this', --
		OPERATOR, '-', OPERATOR, '>', --
		FUNCTION_METHOD, 'bar', --
		OPERATOR, '(', --
		NUMBER, "1'000", OPERATOR, '+', NUMBER, "0xFF'00", OPERATOR, '-', NUMBER, "0b11'00", --
		OPERATOR, ')', --
		OPERATOR, ';', --
		FUNCTION_BUILTIN .. '.stl', 'std::sort', OPERATOR, '(', --
		OPERATOR, '}'
	}
	assert_lex(cpp, code, tags)
end

-- Tests the Markdown lexer with embedded HTML.
function test_markdown()
	local md = lexer.load('markdown')
	local code = ([[

    # header1
    ## header2

    > b1
    > b2
    b3

    1. l1
    2
    * l2
        !code

    t

        code1

    ```
    code2
    ```

    `code3` ``code4`` ``code`5`` `code``6`

        > code7

    ---
    * * *

    [link](target) ![image](target "alt_text") [link] [1]
    http://link text <http://link>
    [1]: link#text

    **strong** *emphasis* \*text\*

    <html>
        </html>

        <a>
  ]]):gsub('\n    ', '\n') -- strip indent
	local tags = {
		HEADING .. '.h1', '# header1', --
		HEADING .. '.h2', '## header2', --
		STRING, '>', DEFAULT, 'b', DEFAULT, '1', --
		STRING, '>', DEFAULT, 'b', DEFAULT, '2', --
		DEFAULT, 'b', DEFAULT, '3', --
		LIST, '1. ', DEFAULT, 'l', DEFAULT, '1', --
		DEFAULT, '2', --
		LIST, '* ', DEFAULT, 'l', DEFAULT, '2', --
		DEFAULT, '!', DEFAULT, 'c', DEFAULT, 'o', DEFAULT, 'd', DEFAULT, 'e', --
		DEFAULT, 't', --
		CODE, 'code1', --
		CODE, '```\ncode2\n```\n', --
		CODE, '`code3`', CODE, '``code4``', CODE, '``code`5``', CODE, '`code``6`', --
		CODE, '> code7', --
		'hr', '---\n', --
		'hr', '* * *\n', --
		LINK, '[link](target)', LINK, '![image](target "alt_text")', REFERENCE, '[link] [1]', --
		LINK, 'http://link', --
		DEFAULT, 't', DEFAULT, 'e', DEFAULT, 'x', DEFAULT, 't', --
		LINK, '<http://link>', --
		REFERENCE, '[1]:', LINK, 'link#text', --
		BOLD, '**strong**', --
		ITALIC, '*emphasis*', --
		DEFAULT, '\\*', DEFAULT, 't', DEFAULT, 'e', DEFAULT, 'x', DEFAULT, 't', DEFAULT, '\\*', --
		TAG .. '.chars', '<', TAG, 'html', TAG .. '.chars', '>', --
		TAG .. '.chars', '</', TAG, 'html', TAG .. '.chars', '>', --
		CODE, '<a>'
	}
	assert_lex(md, code, tags)
end

-- Tests Markdown's custom folder for headers.
function test_markdown_folding()
	local md = lexer.load('markdown')
	local code = [[
		# Level 1

		Description

		## Level 2

		### Level 3

		## Another Level 2

		## Last Level 2

  ]]
	local folds = {1, 5, -7, 9, 11}
	assert_fold_points(md, code, folds)
end

-- Tests the YAML lexer.
function test_yaml()
	local yaml = lexer.load('yaml')

	local code = ([[
    %YAML directive
    --- # document start
    - item 1
      - item 2
        - - item - 3
        - [1, -2.0e-3, 0x3, 04, two words]
    - &anchor !!str item
    - *anchor
    ... # document end
    key: value
    "key 2": 'value 2'
    key 3: value "3"
    key-4_: {1: true, 2: FALSE, 3: null, 4: .Inf, 5: two words, 6: 2000-01-01T12:00:00.0Z}
    - -key - 5: {one: two, three four: five six}
    ? - item 1
    : - item 2
    - {ok: ok@, @: @}
    literal: |
      line 1

      - line 2
      [line, 3]
      #line 4
      ---

    flow: >
      {line: 5}
      ? - line 6
      @line 7
      %line 8
      ...
    - foo: bar
      baz: |
       quux
    - foobar
  ]]):gsub('    ', '') -- strip indent
	local tags = {
		PREPROCESSOR, '%YAML directive', --
		OPERATOR, '---', COMMENT, '# document start', --
		OPERATOR, '-', DEFAULT, 'item 1', --
		OPERATOR, '-', DEFAULT, 'item 2', --
		OPERATOR, '-', OPERATOR, '-', DEFAULT, 'item - 3', --
		OPERATOR, '-', --
		OPERATOR, '[', --
		NUMBER, '1', OPERATOR, ',', --
		NUMBER, '-2.0e-3', OPERATOR, ',', --
		NUMBER, '0x3', OPERATOR, ',', --
		NUMBER, '04', OPERATOR, ',', --
		DEFAULT, 'two words', --
		OPERATOR, ']', --
		OPERATOR, '-', --
		OPERATOR, '&', LABEL, 'anchor', TYPE, '!!str', --
		DEFAULT, 'i', DEFAULT, 't', DEFAULT, 'e', DEFAULT, 'm', --
		OPERATOR, '-', OPERATOR, '*', LABEL, 'anchor', --
		OPERATOR, '...', COMMENT, '# document end', --
		STRING, 'key', OPERATOR, ':', DEFAULT, 'value', --
		STRING, '"key 2"', OPERATOR, ':', STRING, "'value 2'", --
		STRING, 'key 3', OPERATOR, ':', DEFAULT, 'value "3"', --
		STRING, 'key-4_', OPERATOR, ':', OPERATOR, '{', --
		STRING, '1', OPERATOR, ':', CONSTANT_BUILTIN, 'true', OPERATOR, ',', --
		STRING, '2', OPERATOR, ':', CONSTANT_BUILTIN, 'FALSE', OPERATOR, ',', --
		STRING, '3', OPERATOR, ':', CONSTANT_BUILTIN, 'null', OPERATOR, ',', --
		STRING, '4', OPERATOR, ':', NUMBER, '.Inf', OPERATOR, ',', --
		STRING, '5', OPERATOR, ':', DEFAULT, 'two words', OPERATOR, ',', --
		STRING, '6', OPERATOR, ':', NUMBER .. '.timestamp', '2000-01-01T12:00:00.0Z', --
		OPERATOR, '}', --
		OPERATOR, '-', STRING, '-key - 5', OPERATOR, ':', OPERATOR, '{', --
		STRING, 'one', OPERATOR, ':', DEFAULT, 'two', OPERATOR, ',', --
		STRING, 'three four', OPERATOR, ':', DEFAULT, 'five six', --
		OPERATOR, '}', --
		OPERATOR, '?', OPERATOR, '-', DEFAULT, 'item 1', --
		OPERATOR, ':', OPERATOR, '-', DEFAULT, 'item 2', --
		OPERATOR, '-', OPERATOR, '{', --
		STRING, 'ok', OPERATOR, ':', DEFAULT, 'ok@', OPERATOR, ',', --
		ERROR, '@', OPERATOR, ':', ERROR, '@', --
		OPERATOR, '}', --
		STRING, 'literal', OPERATOR, ':', --
		DEFAULT, '|\n  line 1\n\n  - line 2\n  [line, 3]\n  #line 4\n  ---\n', --
		STRING, 'flow', OPERATOR, ':', --
		DEFAULT, '>\n  {line: 5}\n  ? - line 6\n  @line 7\n  %line 8\n  ...', --
		OPERATOR, '-', STRING, 'foo', OPERATOR, ':', DEFAULT, 'bar', --
		STRING, 'baz', OPERATOR, ':', DEFAULT, '|\n   quux', --
		OPERATOR, '-', DEFAULT, 'foobar' --
	}
	assert_lex(yaml, code, tags)

	-- Simulate inserting a newline after |.
	code = ([[
    - foo: bar
      baz: |

       quux
    - foobar
  ]]):gsub('    ', '') -- strip indent
	tags = {
		DEFAULT, '|\n\n   quux', --
		OPERATOR, '-', DEFAULT, 'foobar'
	}
	assert_lex(yaml, code:match('(|.+)$'), tags, DEFAULT)
end

-- Tests the Python lexer and fold by indentation.
function test_python()
	local python = lexer.load('python')

	-- Lexing tests.
	local code = [[
    # Comment.
    @decorator
    class Foo(Bar):
      """documentation"""

      def __init__(self, foo):
        super(Foo, self).__init__()

      def bar(): pass

    if __name__ == '__main__':
      a = -1 + 2.0e3 - 0x4_0 @ 5j
      b = u"foo"
      c = br"\n"
      print(Foo.__doc__)
      Foo().bar()
  ]]
	local tags = {
		COMMENT, '# Comment.', --
		ANNOTATION, '@decorator', --
		KEYWORD, 'class', CLASS, 'Foo', OPERATOR, '(', IDENTIFIER, 'Bar', OPERATOR, ')', OPERATOR, ':', --
		STRING, '"""documentation"""', --
		KEYWORD, 'def', --
		FUNCTION_BUILTIN .. '.special', '__init__', --
		OPERATOR, '(', IDENTIFIER, 'self', OPERATOR, ',', IDENTIFIER, 'foo', OPERATOR, ')', --
		OPERATOR, ':', --
		FUNCTION_BUILTIN, 'super', --
		OPERATOR, '(', --
		IDENTIFIER, 'Foo', --
		OPERATOR, ',', --
		IDENTIFIER, 'self', --
		OPERATOR, ')', --
		OPERATOR, '.', --
		FUNCTION_BUILTIN .. '.special', '__init__', OPERATOR, '(', OPERATOR, ')', --
		KEYWORD, 'def', FUNCTION, 'bar', OPERATOR, '(', OPERATOR, ')', OPERATOR, ':', KEYWORD, 'pass', --
		KEYWORD, 'if', --
		ATTRIBUTE, '__name__', --
		OPERATOR, '=', OPERATOR, '=', --
		STRING, "'__main__'", --
		OPERATOR, ':', --
		IDENTIFIER, 'a', --
		OPERATOR, '=', --
		NUMBER, '-1', --
		OPERATOR, '+', --
		NUMBER, '2.0e3', --
		OPERATOR, '-', --
		NUMBER, '0x4_0', --
		OPERATOR, '@', --
		NUMBER, '5j', --
		IDENTIFIER, 'b', OPERATOR, '=', STRING, 'u"foo"', --
		IDENTIFIER, 'c', OPERATOR, '=', STRING, 'br"\\n"', --
		FUNCTION_BUILTIN, 'print', --
		OPERATOR, '(', --
		IDENTIFIER, 'Foo', OPERATOR, '.', ATTRIBUTE, '__doc__', --
		OPERATOR, ')', --
		FUNCTION, 'Foo', OPERATOR, '(', OPERATOR, ')', --
		OPERATOR, '.', --
		FUNCTION_METHOD, 'bar', OPERATOR, '(', OPERATOR, ')' --
	}
	assert_lex(python, code, tags)

	-- Folding tests.
	code = [[
    class Foo:
      """Documentation
      """
      def foo():
        pass

      def bar(): pass

    if __name__ == '__main__':
      pass
  ]]
	local folds = {1, 4, -6, -8, 9}
	assert_fold_points(python, code, folds)
end

-- Tests output lexer.
function test_output()
	local output = lexer.load('output')
	local text = ([[
    > command
    /tmp/foo:1:2: error
    /tmp/bar.baz:12: warning: warn
    > exit status: 1

    lua: /tmp/quux.lua:34: error
    no error or warning here

    /tmp/foo.sh: line 2: message

    > python3
      File "/tmp/foo.py", line 10

    foo.d(5): Error: no property `foo` for type `Bar`

    "/tmp/foo.plt" line 1: invalid command

    java.lang.Exception: Stack trace
      at com.foo(Foo.java:23)

    No such file or directory in /tmp/foo.php on line 2

    C:\tmp\foo.cs(12, 34): error CS0000: message

    syntax error at /tmp/foo.pl line 7, near "foo"

    CMake Error at CMakeLists.txt:1:
      Parse error.  Expected ...
    -- Configuring incomplete, errors occurred!
  ]]):gsub('    ', '') -- strip indent
	local tags = {
		DEFAULT, '> command', --
		'filename', '/tmp/foo', DEFAULT, ':', --
		'line', '1', DEFAULT, ':', 'column', '2', DEFAULT, ': ', --
		'message', 'error', --
		'filename', '/tmp/bar.baz', DEFAULT, ':', --
		'line', '12', DEFAULT, ': ', --
		'message', 'warning: warn', --
		DEFAULT, '> exit status: 1', --
		--
		DEFAULT, 'lua', DEFAULT, ': ', --
		'filename', '/tmp/quux.lua', DEFAULT, ':', --
		'line', '34', DEFAULT, ': ', --
		'message', 'error', --
		DEFAULT, 'no error or warning here', --
		--
		'filename', '/tmp/foo.sh', DEFAULT, ': ', --
		DEFAULT, 'line ', 'line', '2', DEFAULT, ': ', --
		'message', 'message', --
		--
		DEFAULT, '> python3', --
		DEFAULT, 'File "', 'filename', '/tmp/foo.py', DEFAULT, '", ', --
		DEFAULT, 'line ', 'line', '10', --
		--
		'filename', 'foo.d', DEFAULT, '(', 'line', '5', DEFAULT, ')', DEFAULT, ': ', --
		'message', 'Error: no property `foo` for type `Bar`', --
		--
		DEFAULT, '"', 'filename', '/tmp/foo.plt', DEFAULT, '" ', --
		DEFAULT, 'line ', 'line', '1', DEFAULT, ': ', --
		'message', 'invalid command', --
		--
		DEFAULT, 'java.lang.Exception: Stack trace', --
		DEFAULT, 'at com.foo', DEFAULT, '(', --
		'filename', 'Foo.java', DEFAULT, ':', 'line', '23', DEFAULT, ')', --
		--
		'message', 'No such file or directory', DEFAULT, ' in ', --
		'filename', '/tmp/foo.php', DEFAULT, ' on ', DEFAULT, 'line ', 'line', '2', --
		--
		'filename', 'C:\\tmp\\foo.cs', --
		DEFAULT, '(', 'line', '12', DEFAULT, ', ', 'column', '34', DEFAULT, ')', DEFAULT, ': ', --
		'message', 'error CS0000: message', --
		--
		'message', 'syntax error', DEFAULT, ' at ', --
		'filename', '/tmp/foo.pl', DEFAULT, ' line ', 'line', '7', --
		DEFAULT, ', near "foo"', --
		--
		DEFAULT, 'CMake Error at ', --
		'filename', 'CMakeLists.txt', DEFAULT, ':', 'line', '1', DEFAULT, ':', --
		DEFAULT, 'Parse error.  Expected ...', DEFAULT, '-- Configuring incomplete, errors occurred!'
	}
	assert_lex(output, text, tags)
	local marks = {}
	for _, i in ipairs{2, 6, 9, 12, 14, 16, 19, 21, 23, 25, 27} do marks[i] = 1 end -- errors
	for _, i in ipairs{3} do marks[i] = 2 end -- warnings
	for k, v in pairs(marks) do
		assert(lexer.line_state[k] == v, string.format('line_state[%d] ~= %d', k, v))
	end
	for k, v in pairs(lexer.line_state) do
		assert(marks[k] == v, string.format('marks[%d] ~= %d', k, v))
	end
end

-- Tests LaTeX lexer, particularly its folding.
function test_latex()
	local latex = lexer.load('latex')
	local code = [[
    \begin{document}
      \begin{align} % should inherit environment folding
      E = mc^2
      \end{align}
    \end{document}
  ]]
	local tags = {
		'environment', '\\begin{document}', --
		'environment.math', '\\begin{align}', COMMENT, '% should inherit environment folding', --
		DEFAULT, 'E', DEFAULT, '=', DEFAULT, 'm', DEFAULT, 'c', DEFAULT, '^', DEFAULT, '2', --
		'environment.math', '\\end{align}', --
		'environment', '\\end{document}' --
	}
	assert_lex(latex, code, tags)
	local folds = {1, 2, -4, -5}
	assert_fold_points(latex, code, folds)
end

-- Tests Factor lexer.
function test_factor()
	local factor = lexer.load('factor')

	local code = [[
	#!/usr/bin/env factor
	! comment
	: foo ( a: integer b -- c )
	    { 1.5 3/2 1234 1.2e3 } sum + "hello" length sqrt - ;
	]]

	local tags = {
		COMMENT, '#!/usr/bin/env factor', --
		COMMENT, '! comment', --
		KEYWORD, ':', IDENTIFIER, 'foo', OPERATOR, '( a: integer b -- c )', --
		OPERATOR, '{', NUMBER, '1.5', NUMBER, '3/2', NUMBER, '1234', NUMBER, '1.2e3', OPERATOR, '}', --
		IDENTIFIER, 'sum', --
		OPERATOR, '+', --
		STRING, '\"hello\"', IDENTIFIER, 'length', IDENTIFIER, 'sqrt', --
		OPERATOR, '-', --
		KEYWORD, ';' --
	}
	assert_lex(factor, code, tags)
end

function test_legacy()
	local lex = lexer.new('test')
	local ws = lexer.token(lexer.WHITESPACE, lexer.space^1)
	lex:add_rule('whitespace', ws) -- should call lex:modify_rule()
	assert(lex._rules['whitespace'] == ws)
	lex:add_rule('keyword', lexer.token(KEYWORD, lexer.word_match('foo bar baz')))
	lex:add_rule('number', lexer.token(NUMBER, lexer.number))
	lex:add_rule('preproc', lexer.token(PREPROCESSOR, lexer.starts_line(lexer.to_eol('#'))))
	lex:add_style('whatever', lexer.styles.keyword .. {fore = lexer.colors.red, italic = true})
	local code = "foo 1 bar 2 baz 3\n#quux"
	local tags = {
		KEYWORD, 'foo', --
		NUMBER, '1', --
		KEYWORD, 'bar', --
		NUMBER, '2', --
		KEYWORD, 'baz', --
		NUMBER, '3', --
		PREPROCESSOR, '#quux'
	}
	assert_lex(lex, code, tags)
end

function test_detect()
	assert(lexer.detect('foo.lua') == 'lua')
	assert(lexer.detect('foo.c') == 'c')
	assert(not lexer.detect('foo.txt'))
	assert(lexer.detect('foo', '#!/bin/sh') == 'bash')
	assert(not lexer.detect('foo', '/bin/sh'))

	lexer.detect_extensions.luadoc = 'lua'
	assert(lexer.detect('foo.luadoc') == 'lua')
	assert(lexer.detect('foo.m') == 'objective_c')
	lexer.detect_extensions.m = 'matlab' -- override
	assert(lexer.detect('foo.m') == 'matlab')

	assert(lexer.detect('CMakeLists.txt') == 'cmake') -- not text

	-- Simulate SCI_PRIVATELEXERCALL.
	assert(not lexer.detect()) -- should not error or anything
	lexer.property['lexer.scintillua.filename'] = 'foo.lua'
	assert(lexer.detect() == 'lua')
	lexer.property['lexer.scintillua.line'] = '#!/usr/env/ruby'
	assert(lexer.detect() == 'ruby') -- line detection has priority over filename detection
end

function test_lua51()
	local p = io.popen(
		[[cd lexers && lua5.1 -e 'lexer=require"lexer"' -e 'print(unpack(lexer.load("lua"):lex("_G")))']])
	local output = p:read('a')
	p:close()
	assert(output:find(CONSTANT_BUILTIN .. '\t3', 1, true))
end

-- Run tests.
print('Starting test suite.')
local tests = {}
if #arg == 0 then
	for k, v in pairs(_G) do
		if k:find('^test_') and type(v) == 'function' then tests[#tests + 1] = k end
	end
else
	for i = 1, #arg do if type(_G[arg[i]]) == 'function' then tests[#tests + 1] = arg[i] end end
end
table.sort(tests)
local failed = 0
for i = 1, #tests do
	print(string.format('Running %s.', tests[i]))
	local ok, errmsg = xpcall(_G[tests[i]], function(errmsg)
		print(string.format('Failed!\n%s', debug.traceback(errmsg, 3)))
		failed = failed + 1
	end)
end
print(string.format('%d/%d tests passed', #tests - failed, #tests))
os.exit(failed == 0 and 0 or 1)
