describe("tokenizer", function()
  local tokenizer = require "kdl.v1.tokenizer"

  local function strip(token)
    return { type=token.type, value=token.value }
  end

  it("can peek at upcoming tokens", function()
    local t = tokenizer.new("node 1 2 3")
    assert.same({ type="IDENT", value="node" }, strip(t:peek()))
    assert.same({ type="WS", value=" " }, strip(t:peek_next()))
    assert.same({ type="IDENT", value="node" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:peek()))
    assert.same({ type="INTEGER", value=1 }, strip(t:peek_next()))
  end)

  it("tokenizes identifiers", function()
    assert.same({ type="IDENT", value="foo" }, strip(tokenizer.new("foo"):next()))
    assert.same({ type="IDENT", value="foo-bar123" }, strip(tokenizer.new("foo-bar123"):next()))
  end)

  it("tokenizes strings", function()
    assert.same({ type="STRING", value="foo" }, strip(tokenizer.new('"foo"'):next()))
    assert.same({ type="STRING", value="foo\nbar" }, strip(tokenizer.new('"foo\\nbar"'):next()))
    assert.same({ type="STRING", value="\u{10FFF}" }, strip(tokenizer.new('"\\u{10FFF}"'):next()))
  end)

  it("tokenizes multi line strings", function()
    assert.same({ type="STRING", value="\n  foo\n  bar\n    baz\n  qux\n  " }, strip(tokenizer.new("\"\n  foo\n  bar\n    baz\n  qux\n  \""):next()))
    assert.same({ type="RAWSTRING", value="\n  foo\n  bar\n    baz\n  qux\n  " }, strip(tokenizer.new("r#\"\n  foo\n  bar\n    baz\n  qux\n  \"#"):next()))
  end)

  it("tokenizes rawstrings", function()
    assert.same({ type="RAWSTRING", value="foo\\nbar" }, strip(tokenizer.new('r"foo\\nbar"'):next()))
    assert.same({ type="RAWSTRING", value="foo\"bar" }, strip(tokenizer.new('r#"foo"bar"#'):next()))
    assert.same({ type="RAWSTRING", value="foo\"#bar" }, strip(tokenizer.new('r##"foo"#bar"##'):next()))
    assert.same({ type="RAWSTRING", value="\"foo\"" }, strip(tokenizer.new('r#""foo""#'):next()))

    local t = tokenizer.new('node r#"C:\\Users\\zkat\\"#')
    assert.same({ type="IDENT", value="node" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="RAWSTRING", value="C:\\Users\\zkat\\" }, strip(t:next()))

    t = tokenizer.new('other-node r#"hello"world"#')
    assert.same({ type="IDENT", value="other-node" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="RAWSTRING", value="hello\"world" }, strip(t:next()))
  end)

  it("tokenizes integers", function()
    assert.same({ type="INTEGER", value=0x0123456789abcdef }, strip(tokenizer.new("0x0123456789abcdef"):next()))
    assert.same({ type="INTEGER", value=342391 }, strip(tokenizer.new("0o01234567"):next()))
    assert.same({ type="INTEGER", value=41 }, strip(tokenizer.new("0b101001"):next()))
    assert.same({ type="INTEGER", value=-0x0123456789abcdef }, strip(tokenizer.new("-0x0123456789abcdef"):next()))
    assert.same({ type="INTEGER", value=-342391 }, strip(tokenizer.new("-0o01234567"):next()))
    assert.same({ type="INTEGER", value=-41 }, strip(tokenizer.new("-0b101001"):next()))
    assert.same({ type="INTEGER", value=0x0123456789abcdef }, strip(tokenizer.new("+0x0123456789abcdef"):next()))
    assert.same({ type="INTEGER", value=342391 }, strip(tokenizer.new("+0o01234567"):next()))
    assert.same({ type="INTEGER", value=41 }, strip(tokenizer.new("+0b101001"):next()))
  end)

  it("tokenizes floats", function()
    assert.same({ type="FLOAT", value=1.23 }, strip(tokenizer.new("1.23"):next()))
  end)

  it("tokenizers booleans", function()
    assert.same({ type="TRUE", value=true }, strip(tokenizer.new("true"):next()))
    assert.same({ type="FALSE", value=false }, strip(tokenizer.new("false"):next()))
  end)

  it("tokenizers nulls", function()
    assert.same({ type="NULL", value=nil }, strip(tokenizer.new("null"):next()))
  end)

  it("tokenizers symbols", function()
    assert.same({ type="LBRACE", value="{" }, strip(tokenizer.new("{"):next()))
    assert.same({ type="RBRACE", value="}" }, strip(tokenizer.new("}"):next()))
    assert.same({ type="EQUALS", value="=" }, strip(tokenizer.new("="):next()))
  end)

  it("tokenizes whitespace", function()
    assert.same({ type="WS", value=" " }, strip(tokenizer.new(" "):next()))
    assert.same({ type="WS", value="\t" }, strip(tokenizer.new("\t"):next()))
    assert.same({ type="WS", value="    \t" }, strip(tokenizer.new("    \t"):next()))
    assert.same({ type="WS", value="\\\n" }, strip(tokenizer.new("\\\n"):next()))
    assert.same({ type="WS", value="\\" }, strip(tokenizer.new("\\"):next()))
    assert.same({ type="WS", value="\\\n" }, strip(tokenizer.new("\\//some comment\n"):next()))
    assert.same({ type="WS", value="\\ \n" }, strip(tokenizer.new("\\ //some comment\n"):next()))
    assert.same({ type="WS", value="\\" }, strip(tokenizer.new("\\//some comment"):next()))
  end)

  it("tokenizes multiple tokens", function()
    local t = tokenizer.new("node 1 \"two\" a=3")

    assert.same({ type="IDENT", value="node" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="INTEGER", value=1 }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="STRING", value="two" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="IDENT", value="a" }, strip(t:next()))
    assert.same({ type="EQUALS", value="=" }, strip(t:next()))
    assert.same({ type="INTEGER", value=3 }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes single line comments", function()
    assert.same({ type="EOF", value="" }, strip(tokenizer.new("// comment"):next()))

    local t = tokenizer.new([[node1
// comment
node2]])

    assert.same({ type="IDENT", value="node1" }, strip(t:next()))
    assert.same({ type="NEWLINE", value="\n" }, strip(t:next()))
    assert.same({ type="NEWLINE", value="\n" }, strip(t:next()))
    assert.same({ type="IDENT", value="node2" }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes multiline comments", function()
    local t = tokenizer.new("foo /*bar=1*/ baz=2")

    assert.same({ type="IDENT", value="foo" }, strip(t:next()))
    assert.same({ type="WS", value="  " }, strip(t:next()))
    assert.same({ type="IDENT", value="baz" }, strip(t:next()))
    assert.same({ type="EQUALS", value="=" }, strip(t:next()))
    assert.same({ type="INTEGER", value=2 }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes utf8", function()
    assert.same({ type="IDENT", value="😁" }, strip(tokenizer.new("😁"):next()))
    assert.same({ type="STRING", value="😁" }, strip(tokenizer.new('"😁"'):next()))
    assert.same({ type="IDENT", value="ノード" }, strip(tokenizer.new("ノード"):next()))
    assert.same({ type="IDENT", value="お名前" }, strip(tokenizer.new("お名前"):next()))
    assert.same({ type="STRING", value="☜(ﾟヮﾟ☜)" }, strip(tokenizer.new('"☜(ﾟヮﾟ☜)"'):next()))

    local t = tokenizer.new([[smile "😁"
ノード お名前="☜(ﾟヮﾟ☜)"]])

    assert.same({ type="IDENT", value="smile" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="STRING", value="😁" }, strip(t:next()))
    assert.same({ type="NEWLINE", value="\n" }, strip(t:next()))
    assert.same({ type="IDENT", value="ノード" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="IDENT", value="お名前" }, strip(t:next()))
    assert.same({ type="EQUALS", value="=" }, strip(t:next()))
    assert.same({ type="STRING", value="☜(ﾟヮﾟ☜)" }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes semicolons", function()
    local t = tokenizer.new("node1; node2")

    assert.same({ type="IDENT", value="node1" }, strip(t:next()))
    assert.same({ type="SEMICOLON", value=";" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="IDENT", value="node2" }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes slash dash", function()
    local t = tokenizer.new([[/-mynode /-"foo" /-key=1 /-{
  a
}]])

    assert.same({ type="SLASHDASH", value="/-" }, strip(t:next()))
    assert.same({ type="IDENT", value="mynode" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="SLASHDASH", value="/-" }, strip(t:next()))
    assert.same({ type="STRING", value="foo" }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="SLASHDASH", value="/-" }, strip(t:next()))
    assert.same({ type="IDENT", value="key" }, strip(t:next()))
    assert.same({ type="EQUALS", value="=" }, strip(t:next()))
    assert.same({ type="INTEGER", value=1 }, strip(t:next()))
    assert.same({ type="WS", value=" " }, strip(t:next()))
    assert.same({ type="SLASHDASH", value="/-" }, strip(t:next()))
    assert.same({ type="LBRACE", value="{" }, strip(t:next()))
    assert.same({ type="NEWLINE", value="\n" }, strip(t:next()))
    assert.same({ type="WS", value="  " }, strip(t:next()))
    assert.same({ type="IDENT", value="a" }, strip(t:next()))
    assert.same({ type="NEWLINE", value="\n" }, strip(t:next()))
    assert.same({ type="RBRACE", value="}" }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes multiline nodes", function()
    local t = tokenizer.new([[title \
  "Some title"]])

    assert.same({ type="IDENT", value="title" }, strip(t:next()))
    assert.same({ type="WS", value=" \\\n  " }, strip(t:next()))
    assert.same({ type="STRING", value="Some title" }, strip(t:next()))
    assert.same({ type="EOF", value="" }, strip(t:next()))
    assert.same({ type="EOF", value=nil }, strip(t:next()))
  end)

  it("tokenizes types", function()
    local t = tokenizer.new("(foo)bar")
    assert.same({ type="LPAREN", value="(" }, strip(t:next()))
    assert.same({ type="IDENT", value="foo" }, strip(t:next()))
    assert.same({ type="RPAREN", value=")" }, strip(t:next()))
    assert.same({ type="IDENT", value="bar" }, strip(t:next()))

    t = tokenizer.new("(foo)/*asdf*/bar")
    assert.same({ type="LPAREN", value="(" }, strip(t:next()))
    assert.same({ type="IDENT", value="foo" }, strip(t:next()))
    assert.same({ type="RPAREN", value=")" }, strip(t:next()))
    assert.has_error(function() t:next() end, [[Unexpected '/' (1:6)]])

    t = tokenizer.new("(foo/*asdf*/)bar")
    assert.same({ type="LPAREN", value="(" }, strip(t:next()))
    assert.same({ type="IDENT", value="foo" }, strip(t:next()))
    assert.has_error(function() t:next() end, [[Unexpected '/' (1:5)]])
  end)
end)
