require File.dirname(__FILE__) + "/../../test_helper"

class IndexTest < Test::Unit::TestCase
  include Ferret::Index
  include Ferret::Search
  include Ferret::Analysis
  include Ferret::Store

  def setup()
  end

  def teardown()
  end

  def check_results(index, query, expected)
    cnt = 0
    #puts "#{query} - #{expected.inspect}"
    #puts index.size
    index.search_each(query) do |doc, score|
      #puts "doc-#{doc} score=#{score}"
      assert_not_nil(expected.index(doc), "doc #{doc} found but not expected")
      cnt += 1
    end
    assert_equal(expected.length, cnt)
  end

  def do_test_index_with_array(index)
    [
      ["one two"],
      ["one", "three"],
      ["two"],
      ["one", "four"],
      ["one two"],
      ["two", "three", "four"],
      ["one"],
      ["two", "three", "four", "five"]
    ].each {|doc| index << doc }
    assert_equal(8, index.size)
    q = "one"
    check_results(index, q, [0, 1, 3, 4, 6])
    q = "one AND two"
    check_results(index, q, [0, 4])
    q = "one OR five"
    check_results(index, q, [0, 1, 3, 4, 6, 7])
    assert_equal(%w{two three four five}, index.doc(7)[:xxx])
  end

  def do_test_index_with_hash(index)
    data = [
      {:xxx => "one two"},
      {:xxx => "one", :field2 => "three"},
      {:xxx => "two"},
      {:xxx => "one", :field2 => "four"},
      {:xxx => "one two"},
      {:xxx => "two", :field2 => "three", :field3 => "four"},
      {:xxx => "one"},
      {:xxx => "two", :field2 => "three", :field3 => "five"}
    ]
    data.each {|doc| index << doc }
    q = "one AND two"
    check_results(index, q, [0, 4])
    q = "one OR five"
    check_results(index, q, [0, 1, 3, 4, 6])
    q = "one OR field3:five"
    check_results(index, q, [0, 1, 3, 4, 6, 7])
    assert_equal("four", index[5]["field3"])
    q = "field3:f*"
    check_results(index, q, [5, 7])
    q = "*:(one AND NOT three)"
    check_results(index, q, [0, 3, 4, 6])
    q = "*:(one AND (NOT three))"
    check_results(index, q, [0, 3, 4, 6])
    q = "two AND field3:f*"
    check_results(index, q, [5, 7])
    assert_equal("five", index.doc(7)["field3"])
    assert_equal("two", index.doc(7)[:xxx])
  end

  def do_test_index_with_doc_array(index)
    data = [
      {:xxx => "one two multi", :id => "myid"},
      {:xxx => "one", :field2 => "three multi"},
      {:xxx => "two"},
      {:xxx => "one", :field2 => "four"},
      {:xxx => "one two"},
      {:xxx => "two", :field2 => "three", :field3 => "four"},
      {:xxx => "one multi2", :id => "hello"},
      {:xxx => "two", :field2 => "this three multi2", :field3 => "five multi"}
    ]
    data.each {|doc| index << doc }
    q = "one AND two"
    check_results(index, q, [0, 4])
    q = "one OR five"
    check_results(index, q, [0, 1, 3, 4, 6])
    q = "one OR field3:five"
    check_results(index, q, [0, 1, 3, 4, 6, 7])
    q = "two AND (field3:f*)"
    check_results(index, q, [5, 7])
    q = "*:(multi OR multi2)"
    check_results(index, q, [0, 1, 6, 7])
    q = "field2|field3:(multi OR multi2)"
    check_results(index, q, [1, 7])
    doc = index[5]
    assert_equal("three", index[5]["field2"])
    assert(!index.has_deletions?)
    assert(!index.deleted?(5))
    assert_equal(8, index.size)
    index.delete(5)
    assert(index.has_deletions?)
    assert(index.deleted?(5))
    assert_equal(7, index.size)
    q = "two AND (field3:f*)"
    check_results(index, q, [7])
    
    doc.load
    doc[:field2] = "dave"
    index << doc
    check_results(index, q, [7, 8])
    check_results(index, "*:this", [])
    assert_equal(8, index.size)
    assert_equal("dave", index[8][:field2])
    index.optimize
    check_results(index, q, [6, 7])
    assert_equal("dave", index[7][:field2])
    index.query_delete("field2:three")
    assert(index.deleted?(1))
    assert(index.deleted?(6))
    assert(! index.deleted?(7))
    assert_equal("one multi2", index["hello"][:xxx])
    assert_equal("one two multi", index["myid"][:xxx])
    index.delete("myid")
    assert(index.deleted?(0))
  end

  def test_ram_index
    index = Ferret::I.new(:default_input_field => :xxx)
    do_test_index_with_array(index)
    index.close

    index = Index.new(:default_field => :xxx)
    do_test_index_with_hash(index)
    index.close

    index = Index.new(:default_field => :xxx, :id_field => :id)
    do_test_index_with_doc_array(index)
    index.close
  end

  def test_fs_index
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))

    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
    assert_raise(Ferret::FileNotFoundError) do
      Index.new(:path => fs_path,
                :create_if_missing => false,
                :default_field => :xxx)
    end

    index = Index.new(:path => fs_path, :default_input_field => :xxx)
    do_test_index_with_array(index)
    index.close

    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
    index = Index.new(:path => fs_path, :default_field => :xxx)
    do_test_index_with_hash(index)
    index.close

    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
    index = Index.new(:path => fs_path,
                      :default_field => :xxx,
                      :id_field => "id")
    do_test_index_with_doc_array(index)
    index.close
  end

  def test_fs_index_is_persistant
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
    index = Index.new(:path => fs_path, :default_field => :xxx, :create => true)

    [
      {:xxx => "one two", :id => "me"},
      {:xxx => "one", :field2 => "three"},
      {:xxx => "two"},
      {:xxx => "one", :field2 => "four"},
      {:xxx => "one two"},
      {:xxx => "two", :field2 => "three", :field3 => "four"},
      {:xxx => "one"},
      {:xxx => "two", :field2 => "three", :field3 => "five"}
    ].each {|doc| index << doc }
    assert_equal(8, index.size)
    index.close

    index = Index.new(:path => fs_path, :create_if_missing => false)
    assert_equal(8, index.size)
    assert_equal("four", index[5]["field3"])
    index.close
  end

  def test_key_used_for_id_field
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir2'))

    index = Index.new(:path => fs_path, :key => :my_id, :create => true)
    [
      {:my_id => "three", :id => "me"},
      {:my_id => "one", :field2 => "three"},
      {:my_id => "two"},
      {:my_id => "one", :field2 => "four"},
      {:my_id => "three"},
      {:my_id => "two", :field2 => "three", :field3 => "four"},
      {:my_id => "one"},
      {:my_id => "two", :field2 => "three", :field3 => "five"}
    ].each {|doc| index << doc }
    index.optimize
    assert_equal(3, index.size)
    assert_equal("three", index["two"][:field2])
    index.close
  end

  def test_merging_indexes
    index1 = Index.new(:default_field => :f)
    index2 = Index.new(:default_field => :f)
    index3 = Index.new(:default_field => :f)

    [
      {:f => "zero"},
      {:f => "one"},
      {:f => "two"}
    ].each {|doc| index1 << doc }
    [
      {:f => "three"},
      {:f => "four"},
      {:f => "five"}
    ].each {|doc| index2 << doc }
    [
      {:f => "six"},
      {:f => "seven"},
      {:f => "eight"}
    ].each {|doc| index3 << doc }

    index = Index.new(:default_field => :f)
    index.add_indexes(index1)
    assert_equal(3, index.size)
    assert_equal("zero", index[0][:f])
    index.add_indexes([index2, index3])
    assert_equal(9, index.size)
    assert_equal("zero", index[0][:f])
    assert_equal("eight", index[8][:f])
    index1.close
    index2.close
    index3.close
    assert_equal("seven", index[7][:f])
    data = [
      {:f => "alpha"},
      {:f => "beta"},
      {:f => "charlie"}
    ]
    dir1 = RAMDirectory.new
    index1 = Index.new(:dir => dir1, :default_field => :f)
    data.each {|doc| index1 << doc }
    index1.flush
    data = [
      {:f => "delta"},
      {:f => "echo"},
      {:f => "foxtrot"}
    ]
    dir2 = RAMDirectory.new
    index2 = Index.new(:dir => dir2, :default_field => :f)
    data.each {|doc| index2 << doc }
    index2.flush
    data = [
      {:f => "golf"},
      {:f => "india"},
      {:f => "juliet"}
    ]
    dir3 = RAMDirectory.new
    index3 = Index.new(:dir => dir3, :default_field => :f)
    data.each {|doc| index3 << doc }
    index3.flush

    index.add_indexes(dir1)
    assert_equal(12, index.size)
    assert_equal("alpha", index[9][:f])
    index.add_indexes([dir2, dir3])
    assert_equal(18, index.size)
    assert_equal("juliet", index[17][:f])
    index1.close
    dir1.close
    index2.close
    dir2.close
    index3.close
    dir3.close
    assert_equal("golf", index[15][:f])
    index.close
  end

  def test_persist_index
    data = [
      {:f => "zero"},
      {:f => "one"},
      {:f => "two"}
    ]
    index = Index.new(:default_field => :f)
    data.each {|doc| index << doc }
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))

    index.persist(fs_path, true)
    assert_equal(3, index.size)
    assert_equal("zero", index[0][:f])
    index.close

    index = Index.new(:path => fs_path)
    assert_equal(3, index.size)
    assert_equal("zero", index[0][:f])
    index.close


    data = [
      {:f => "romeo"},
      {:f => "sierra"},
      {:f => "tango"}
    ]
    index = Index.new(:default_field => :f)
    data.each {|doc| index << doc }
    assert_equal(3, index.size)
    assert_equal("romeo", index[0][:f])
    dir = FSDirectory.new(fs_path, false)
    index.persist(dir)
    assert_equal(6, index.size)
    assert_equal("zero", index[0][:f])
    assert_equal("romeo", index[3][:f])
    index.close

    index = Index.new(:path => fs_path)
    assert_equal(6, index.size)
    assert_equal("zero", index[0][:f])
    assert_equal("romeo", index[3][:f])
    index.close
  end

  def test_auto_update_when_externally_modified()
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
    index = Index.new(:path => fs_path, :default_field => :f, :create => true)
    index << "document 1"
    assert_equal(1, index.size)

    index2 = Index.new(:path => fs_path, :default_field => :f)
    assert_equal(1, index2.size)
    index2 << "document 2"
    assert_equal(2, index2.size)
    assert_equal(2, index.size)
    top_docs = index.search("content3")

    assert_equal(0, top_docs.hits.size)

    iw = IndexWriter.new(:path => fs_path, :analyzer => WhiteSpaceAnalyzer.new)
    iw << {:f => "content3"}
    iw.close()

    top_docs = index.search("content3")
    assert_equal(1, top_docs.hits.size)
    assert_equal(3, index.size)
    assert_equal("content3", index[2][:f])
    index2.close
    index.close
  end

  def test_delete
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
    [
      {:id => 0, :cat => "/cat1/subcat1"},
      {:id => 1, :cat => "/cat1/subcat2"},
      {:id => 2, :cat => "/cat1/subcat2"},
      {:id => 3, :cat => "/cat1/subcat3"},
      {:id => 4, :cat => "/cat1/subcat4"},
      {:id => 5, :cat => "/cat2/subcat1"},
      {:id => 6, :cat => "/cat2/subcat2"},
      {:id => 7, :cat => "/cat2/subcat3"},
      {:id => 8, :cat => "/cat2/subcat4"},
      {:id => 9, :cat => "/cat2/subcat5"},
    ].each {|doc| index << doc }
    assert_equal(10, index.size)
    assert_equal(1, index.search("id:9").total_hits)
    index.delete(9)
    assert_equal(9, index.size)
    assert_equal(0, index.search("id:9").total_hits)
    assert_equal(1, index.search("id:8").total_hits)
    index.delete("8")
    assert_equal(8, index.size)
    assert_equal(0, index.search("id:8").total_hits)
    assert_equal(5, index.search("cat:/cat1*").total_hits)
    index.query_delete("cat:/cat1*")
    assert_equal(3, index.size)
    assert_equal(0, index.search("cat:/cat1*").total_hits)
    index.close
  end

  def test_update
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                      :default_input_field => :content,
                      :id_field => :id)
    [
      {:id => 0, :cat => "/cat1/subcat1", :content => "content0"},
      {:id => 1, :cat => "/cat1/subcat2", :content => "content1"},
      {:id => 2, :cat => "/cat1/subcat2", :content => "content2"},
      {:id => 3, :cat => "/cat1/subcat3", :content => "content3"},
      {:id => 4, :cat => "/cat1/subcat4", :content => "content4"},
      {:id => 5, :cat => "/cat2/subcat1", :content => "content5"},
      {:id => 6, :cat => "/cat2/subcat2", :content => "content6"},
      {:id => 7, :cat => "/cat2/subcat3", :content => "content7"},
      {:id => 8, :cat => "/cat2/subcat4", :content => "content8"},
      {:id => 9, :cat => "/cat2/subcat5", :content => "content9"},
    ].each { |doc| index << doc }
    assert_equal(10, index.size)
    assert_equal("content5", index["5"][:content])
    index.query_update("id:5", {:content => "content five"})
    assert_equal("content five", index["5"][:content])
    assert_equal(nil, index["5"][:extra_content])
    index.update("5", {:id => "5",
                       :cat => "/cat1/subcat6",
                       :content => "high five",
                       :extra_content => "hello"})
    assert_equal("hello", index["5"][:extra_content])
    assert_equal("high five", index["5"][:content])
    assert_equal("/cat1/subcat6", index["5"][:cat])
    assert_equal("content9", index["9"][:content])
    index.query_update("content:content9", {:content => "content nine"})
    assert_equal("content nine", index["9"][:content])
    assert_equal("content0", index["0"][:content])
    assert_equal(nil, index["0"][:extra_content])
    document = index[0].load
    document[:content] = "content zero"
    document[:extra_content] = "extra content"
    index.update(0, document)
    assert_equal("content zero", index["0"][:content])
    assert_equal("extra content", index["0"][:extra_content])
    assert_equal(nil, index["1"][:tag])
    assert_equal(nil, index["2"][:tag])
    assert_equal(nil, index["3"][:tag])
    assert_equal(nil, index["4"][:tag])
    index.query_update("id:<5 AND cat:>=/cat1/subcat2", {:tag => "cool"})
    assert_equal("cool", index["1"][:tag])
    assert_equal("cool", index["2"][:tag])
    assert_equal("cool", index["3"][:tag])
    assert_equal("cool", index["4"][:tag])
    assert_equal(4, index.search("tag:cool").total_hits)
    index.close
  end

  def test_index_key
    data = [
      {:id => 0, :val => "one"},
      {:id => 0, :val => "two"},
      {:id => 1, :val => "three"},
      {:id => 1, :val => "four"},
    ]
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                      :key => :id)
    data.each { |doc| index << doc }
    assert_equal(2, index.size)
    assert_equal("two", index["0"][:val])
    assert_equal("four", index["1"][:val])
    index.close
  end

  def test_index_key_batch0
    data = {
      #"0" => {:id => "0", :val => "one"},
      "0" => {:id => "0", :val => "two"},
      #"1" =>{:id => "1", :val => "three"},
      "1" => {:id => "1", :val => "four"},
    }

    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                      :key => :id)
    index.batch_update data
    assert_equal(2, index.size)
    index.close
  end

  def test_index_key_batch1
    data0 = {
      #"0" => {:id => "0", :val => "one"},
      "0" => {:id => "0", :val => "two"},
      "1" =>{:id => "1", :val => "three"},
      "2" => {:id => "1", :val => "four"},
    }

    data1 = {
      "0" => {:id => "0", :val => "one"},
      "3" => {:id => "3", :val => "two"},
      "2" =>{:id => "2", :val => "three"},
      "1" => {:id => "1", :val => "four"},
      "4" => {:id => "4", :val => "four"},
    }

    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                      :key => :id)
    index.batch_update data0
    assert_equal(3, index.size)
    index.batch_update data1
    assert_equal(5, index.size)
    index.close
  end

  def test_index_key_delete_batch0
    data0 = {
      "1" =>{:id => "1", :val => "three"},
      "2" => {:id => "2", :val => "four"},
      "0" => {:id => "0", :val => "four"},
    }

    data1 = ["0", "1"];

    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new, :key => :id)
    index.batch_update data0

    assert_equal("four", index["0"][:val])
    assert_equal("three", index["1"][:val])
    assert_equal("four", index["2"][:val])

    assert_equal(3, index.size)
    index.delete data1
    assert_equal(1, index.size)
    assert_equal("four", index["2"][:val])

    index.close
  end

  def test_index_key_delete_batch1
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
    1000.times {|i| index << {:id => "#{i}", :content => "content #{i}"}}
    assert_equal(1000, index.size)
    assert_equal("content 876", index['876'][:content])

    new_docs = Array.new(1000) {|i| {:id => i, :content => "#{i} > content"}}
    index.batch_update(new_docs)
    assert_equal(1000, index.size)
    assert_equal("128 > content", index['128'][:content])

    new_docs = Array.new(1000) {|i| {:id => i.to_s, :content => "_(#{i})_"}}
    index.batch_update(new_docs)
    assert_equal(1000, index.size)
    assert_equal("_(287)_", index['287'][:content])

    new_docs = {}
    1000.times {|i| new_docs[i.to_s] = {:id => i, :content => "Hash(#{i})"}}
    index.batch_update(new_docs)
    assert_equal(1000, index.size)
    assert_equal("Hash(78)", index['78'][:content])
  end

  def test_index_multi_key
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                      :key => [:id, :table])
    [
      {:id => 0, :table => "product", :product => "tent"},
      {:id => 0, :table => "location", :location => "first floor"},
      {:id => 0, :table => "product", :product => "super tent"},
      {:id => 0, :table => "location", :location => "second floor"},
      {:id => 1, :table => "product", :product => "backback"},
      {:id => 1, :table => "location", :location => "second floor"},
      {:id => 1, :table => "location", :location => "first floor"},
      {:id => 1, :table => "product", :product => "rucksack"},
      {:id => 1, :table => "product", :product => "backpack"}
    ].each { |doc| index << doc }
    index.optimize
    assert_equal(4, index.size)
    assert_equal("super tent", index[0][:product])
    assert_equal("second floor", index[1][:location])
    assert_equal("backpack", index[3][:product])
    assert_equal("first floor", index[2][:location])
    index.close
  end

  def test_index_multi_key_untokenized
    field_infos = FieldInfos.new(:term_vector => :no)
    field_infos.add_field(:id, :index => :untokenized)
    field_infos.add_field(:table, :index => :untokenized)

    index = Index.new(:analyzer => Analyzer.new,
                      :key => [:id, :table],
                      :field_infos => field_infos)
    [
      {:id => 0, :table => "Product", :product => "tent"},
      {:id => 0, :table => "location", :location => "first floor"},
      {:id => 0, :table => "Product", :product => "super tent"},
      {:id => 0, :table => "location", :location => "second floor"},
      {:id => 1, :table => "Product", :product => "backback"},
      {:id => 1, :table => "location", :location => "second floor"},
      {:id => 1, :table => "location", :location => "first floor"},
      {:id => 1, :table => "Product", :product => "rucksack"},
      {:id => 1, :table => "Product", :product => "backpack"}
    ].each {|doc| index << doc}

    assert_equal(4, index.size)
    index.optimize
    assert_equal("super tent", index[0][:product])
    assert_equal("second floor", index[1][:location])
    assert_equal("backpack", index[3][:product])
    assert_equal("first floor", index[2][:location])
    index.close
  end

  def test_sortby_date
    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)

    [
      {:content => "one", :date => "20051023"},
      {:content => "two", :date => "19530315"},
      {:content => "three four", :date => "19390912"},
      {:content => "one", :date => "19770905"},
      {:content => "two", :date => "19810831"},
      {:content => "three", :date => "19790531"},
      {:content => "one", :date => "19770725"},
      {:content => "two", :date => "19751226"},
      {:content => "four", :date => "19390912"}
    ].each {|doc| index << doc}

    sf_date = SortField.new("date", {:type => :integer})
    #top_docs = index.search("one", :sort => [sf_date, SortField::SCORE])
    top_docs = index.search("one", :sort => Sort.new("date"))
    assert_equal(3, top_docs.total_hits)
    assert_equal("19770725", index[top_docs.hits[0].doc][:date])
    assert_equal("19770905", index[top_docs.hits[1].doc][:date])
    assert_equal("20051023", index[top_docs.hits[2].doc][:date])
    top_docs = index.search("one two three four",
                            :sort => [sf_date, SortField::SCORE])

    assert_equal("19390912", index[top_docs.hits[0].doc][:date])
    assert_equal("three four", index[top_docs.hits[0].doc][:content])
    assert_equal("19390912", index[top_docs.hits[1].doc][:date])
    assert_equal("four", index[top_docs.hits[1].doc][:content])
    assert_equal("19530315", index[top_docs.hits[2].doc][:date])

    top_docs = index.search("one two three four",
                            :sort => [:date, :content])
    assert_equal("19390912", index[top_docs.hits[0].doc][:date])
    assert_equal("four", index[top_docs.hits[0].doc][:content])
    assert_equal("19390912", index[top_docs.hits[1].doc][:date])
    assert_equal("three four", index[top_docs.hits[1].doc][:content])
    assert_equal("19530315", index[top_docs.hits[2].doc][:date])

    index.close
  end
  
  # this test has been corrected to work as intended
  # it now fails the same way on both 1.8 and 1.9 -- sds
  def test_auto_flush
    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}

    data = %w(one two three four five six seven eight nine ten eleven twelve)
    index1 = Index.new(:path => fs_path, :auto_flush => true, :key => :id)
    index1 << {:id => 0, :content => "zero"}
    index2 = Index.new(:path => fs_path, :auto_flush => true)
    begin
      n = 1
      data.each do |datum|
        index1 << {:id => n, :content => datum}
        index2 << {:id => n, :content => datum}
        n += 1
      end
      5.times do |i|
        index1.delete(i)
        index2.delete(i + 5)
      end
      index1.optimize
      index2 << "thirteen"
    rescue Exception
      assert(false, "This should not cause an error when auto flush has been set")
    end
    index1.close
    index2.close
  end

  def test_doc_specific_analyzer
    index = Index.new
    index.add_document("abc", Ferret::Analysis::Analyzer.new)
    assert_equal(1, index.size)
  end

  def test_adding_empty_term_vectors
    index = Index.new(:field_infos => FieldInfos.new(:term_vector => :no))

    # Note: Adding keywords to either field1 or field2 gets rid of the error

    index << {:field1 => ''}
    index << {:field2 => ''}
    index << {:field3 => 'foo bar baz'}

    index.flush
    index.close
  end

  def test_stopwords
    field_infos = FieldInfos.new(:store => :no, :term_vector => :no)
    field_infos.add_field(:id, :store => :yes, :index => :untokenized)

    i = Ferret::Index::Index.new(:or_default => false, :default_search_field => '*')
 
    # adding this additional field to the document leads to failure below
    # comment out this statement and all tests pass:
    i << {:id => 1, :content => "Move or shake"}

    hits = i.search 'move nothere shake'
    assert_equal 0, hits.total_hits
    hits = i.search 'move shake'
    assert_equal 1, hits.total_hits
    hits = i.search 'move or shake'
    assert_equal 1, hits.total_hits # fails when id field is present
  end

  def test_threading
    path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
    index = Ferret::Index::Index.new(:path => path, :create => true)

    100.times do |i|
      doc = {}
      doc[:id] = i
      doc[:foo] = "foo #{i}"
      index << doc
    end

    threads = []

    4.times do
      threads << Thread.new(index) do |idx|
        result = idx.search('id:42')
        assert_equal(1, result.total_hits)
      end
    end

    threads.each{|t| t.join }
  end 

  def test_wildcard
    j = nil
    Ferret::I.new do |i|
      i << "one"
      assert_equal(1, i.search("*").total_hits)
      i << "two"
      assert_equal(2, i.search("*").total_hits)
      i << {:content => "three"}
      assert_equal(3, i.search("*").total_hits)
      assert_equal(3, i.search("id:*").total_hits)
      assert_equal(2, i.search('id:?*').total_hits)
      j = i
    end
    assert_raise(StandardError) {j.close} 
  end

  def check_highlight(index, q, excerpt_length, num_excerpts, expected, field = :field)
    highlights = index.highlight(q, 0,
                                 :excerpt_length => excerpt_length,
                                 :num_excerpts => num_excerpts,
                                 :field => field)
    assert_equal(expected, highlights)
    highlights = index.highlight(q, 1,
                                 :excerpt_length => excerpt_length,
                                 :num_excerpts => num_excerpts,
                                 :field => field)
    assert_equal(expected, highlights)
  end

  def test_highlighter()
    index = Ferret::I.new(:default_field => :field,
                          :default_input_field => :field,
                          :analyzer => Ferret::Analysis::WhiteSpaceAnalyzer.new)
    [
      "the words we are searching for are one and two also " +
      "sometimes looking for them as a phrase like this; one " +
      "two lets see how it goes",
      [
        "the words we",
        "are searching",
        "for are one",
        "and two also",
        "sometimes looking",
        "for them as a",
        "phrase like this;",
        "one two lets see",
        "how it goes"
      ]
    ].each {|doc| index << doc }
    
    check_highlight(index, "one", 10, 1, ["...are <b>one</b>..."])
    check_highlight(index, "one", 10, 2,
                    ["...are <b>one</b>...","...this; <b>one</b>..."])
    check_highlight(index, "one", 10, 3,
                    ["the words...","...are <b>one</b>...","...this; <b>one</b>..."])
    check_highlight(index, "one", 10, 4,
                    ["the words we are...","...are <b>one</b>...","...this; <b>one</b>..."])
    check_highlight(index, "one", 10, 5,
                    ["the words we are searching for are <b>one</b>...","...this; <b>one</b>..."])
    check_highlight(index, "one", 10, 20,
                    ["the words we are searching for are <b>one</b> and two also " +
                     "sometimes looking for them as a phrase like this; <b>one</b> " +
                     "two lets see how it goes"])
    check_highlight(index, "one", 200, 1,
                    ["the words we are searching for are <b>one</b> and two also " +
                     "sometimes looking for them as a phrase like this; <b>one</b> " +
                     "two lets see how it goes"])
    check_highlight(index, "(one two)", 15, 2, 
                    ["...<b>one</b> and <b>two</b>...","...this; <b>one</b> <b>two</b>..."])
    check_highlight(index, 'one two "one two"', 15, 2, 
                    ["...<b>one</b> and <b>two</b>...","...this; <b>one two</b>..."])
    check_highlight(index, 'one two "one two"', 15, 1, 
                    ["...this; <b>one two</b>..."])
    check_highlight(index, '"one two"', 15, 1, nil, :not_a_field)
    check_highlight(index, 'wrong_field:one', 15, 1, nil, :wrong_field)
    check_highlight(index, '"the words" "for are one and two" words one two', 10, 1, 
                    ["<b>the words</b>..."])
    check_highlight(index, '"the words" "for are one and two" words one two', 20, 2, 
                    ["<b>the words</b> we are...","...<b>for are one and two</b>..."])
    index.close
  end

  def test_changing_analyzer
    index = Ferret::I.new
    a = Ferret::Analysis::WhiteSpaceAnalyzer.new(false)
    index.add_document({:content => "Content With Capitals"}, a)
    tv = index.reader.term_vector(0, :content)
    assert_equal("Capitals", tv.terms[0].text)
    index.close
  end

  def test_top_doc_to_json
    index = Ferret::I.new
    [
      {:f1 => "one"},
      {:f2 => ["two",2,2.0]},
      {:f3 => 3},
      {:f4 => 4.0},
      {:f5 => "five", :funny => '"' * 10_000}
    ].each {|doc| index << doc}
    json_str = index.search("one two 3 4.0 five",
                            :sort => Ferret::Search::Sort::INDEX_ORDER).to_json
    assert(json_str == '[{"f1":"one"},{"f2":["two","2","2.0"]},{"f3":"3"},{"f4":"4.0"},{"f5":"five","funny":"' + '\'"\'' * 10_000 + '"}]' ||
           json_str == '[{"f1":"one"},{"f2":["two","2","2.0"]},{"f3":"3"},{"f4":"4.0"},{"funny":"' + '\'"\'' * 10_000 + '","f5":"five"}]')
    assert_equal('[]', index.search("xxx").to_json)
    index.close
  end

  def test_large_query_delete
    index = Ferret::I.new
    20.times do
      index << {:id => 'one'}
      index << {:id => 'two'}
    end
    index.query_delete('id:one')
    assert_equal(20, index.size)
  end

  def test_query_update_delete_more_than_ten
    index = Ferret::I.new
    20.times {|i| index << {:id => i, :find => 'match', :change => 'one'} }

    assert_equal(20, index.search('find:match').total_hits)
    index.query_update('find:match', {:change => 'two'})
    assert_equal(20, index.search('find:match AND change:two').total_hits)
    index.query_delete('find:match')
    assert_equal(0, index.size)
  end
end
