require 'strscan'

module HH::Syntax

  # A single token extracted by a tokenizer. It is simply the lexeme
  # itself, decorated with a 'group' attribute to identify the type of the
  # lexeme.
  class Token < String

    # the type of the lexeme that was extracted.
    attr_reader :group

    # the instruction associated with this token (:none, :region_open, or
    # :region_close)
    attr_reader :instruction

    # Create a new Token representing the given text, and belonging to the
    # given group.
    def initialize( text, group, instruction = :none )
      super text
      @group = group
      @instruction = instruction
    end

  end

  # The base class of all tokenizers. It sets up the scanner and manages the
  # looping until all tokens have been extracted. It also provides convenience
  # methods to make sure adjacent tokens of identical groups are returned as
  # a single token.
  class Tokenizer

    # The current group being processed by the tokenizer
    attr_reader :group

    # The current chunk of text being accumulated
    attr_reader :chunk

    # Start tokenizing. This sets up the state in preparation for tokenization,
    # such as creating a new scanner for the text and saving the callback block.
    # The block will be invoked for each token extracted.
    def start( text, &block )
      @chunk = ""
      @group = :normal
      @callback = block
      @text = StringScanner.new( text )
      setup
    end

    # Subclasses may override this method to provide implementation-specific
    # setup logic.
    def setup
    end

    # Finish tokenizing. This flushes the buffer, yielding any remaining text
    # to the client.
    def finish
      start_group nil
      teardown
    end

    # Subclasses may override this method to provide implementation-specific
    # teardown logic.
    def teardown
    end

    # Subclasses must implement this method, which is called for each iteration
    # of the tokenization process. This method may extract multiple tokens.
    def step
      raise NotImplementedError, "subclasses must implement #step"
    end

    # Begins tokenizing the given text, calling #step until the text has been
    # exhausted.
    def tokenize( text, &block )
      start text, &block
      step until @text.eos?
      finish
    end

    # Specify a set of tokenizer-specific options. Each tokenizer may (or may
    # not) publish any options, but if a tokenizer does those options may be
    # used to specify optional behavior.
    def set( opts={} )
      ( @options ||= Hash.new ).update opts
    end

    # Get the value of the specified option.
    def option(opt)
      @options ? @options[opt] : nil
    end

  private

    EOL = /(?=\r\n?|\n|$)/

    # A convenience for delegating method calls to the scanner.
    def self.delegate( sym )
      define_method( sym ) { |*a| @text.__send__( sym, *a ) }
    end

    delegate :bol?
    delegate :eos?
    delegate :scan
    delegate :scan_until
    delegate :check
    delegate :check_until
    delegate :getch
    delegate :matched
    delegate :pre_match
    delegate :peek
    delegate :pos

    # Access the n-th subgroup from the most recent match.
    def subgroup(n)
      @text[n]
    end

    # Append the given data to the currently active chunk.
    def append( data )
      @chunk << data
    end

    # Request that a new group be started. If the current group is the same
    # as the group being requested, a new group will not be created. If a new
    # group is created and the current chunk is not empty, the chunk's
    # contents will be yielded to the client as a token, and then cleared.
    #
    # After the new group is started, if +data+ is non-nil it will be appended
    # to the chunk.
    def start_group( gr, data=nil )
      flush_chunk if gr != @group
      @group = gr
      @chunk << data if data
    end

    def start_region( gr, data=nil )
      flush_chunk
      @group = gr
      @callback.call( Token.new( data||"", @group, :region_open ) )
    end

    def end_region( gr, data=nil )
      flush_chunk
      @group = gr
      @callback.call( Token.new( data||"", @group, :region_close ) )
    end

    def flush_chunk
      @callback.call( Token.new( @chunk, @group ) ) unless @chunk.empty?
      @chunk = ""
    end

    def subtokenize( syntax, text )
      tokenizer = Syntax.load( syntax )
      tokenizer.set @options if @options
      flush_chunk
      tokenizer.tokenize( text, &@callback )
    end
  end


  # A default tokenizer for handling syntaxes that are not explicitly handled
  # elsewhere. It simply yields the given text as a single token.
  class Default

    # Yield the given text as a single token.
    def tokenize( text )
      yield Token.new( text, :normal )
    end

  end

  # A hash for registering syntax implementations.
  SYNTAX = Hash.new( Default )

  # Load the implementation of the requested syntax. If the syntax cannot be
  # found, or if it cannot be loaded for whatever reason, the Default syntax
  # handler will be returned.
  def load( syntax )
    begin
      require_relative "lang/#{syntax}"
    rescue LoadError
    end
    SYNTAX[ syntax ].new
  end
  module_function :load

  # Return an array of the names of supported syntaxes.
  def all
    lang_dir = File.join(File.dirname(__FILE__), "syntax", "lang")
    Dir["#{lang_dir}/*.rb"].map { |path| File.basename(path, ".rb") }
  end
  module_function :all


end
