$LOAD_PATH.unshift '/srv/whimsy/lib'

require 'find'
require 'weakref'
require 'whimsy/asf/config'

module ASF
  # utility methods for handling mailing list attributes:
  # - subscriptions
  # - moderation
  module MLIST

    # whilst the source files are not particularly difficult to parse, it makes
    # sense to centralise access so any necessary changes can be localised

    # Note that email matching is case blind, but the original case is returned
    # list and domain names are always returned as lower-case

    # Potentially also the methods could check if access was allowed.
    # This is currently done by the callers

    # Note that the data files don't provide information on whether a list is
    # public or private.

    @@file_times  = {} # Key=type, value = modtime
    @@file_parsed = {} # Key=type, value = cache hash

    # Return an array of board subscribers followed by the file update time
    def self.board_subscribers(archivers=true)
      return list_filter('sub', 'apache.org', 'board', archivers), File.mtime(LIST_TIME)
    end

    # Return an array of members@ subscribers followed by the file update time
    def self.members_subscribers(archivers=true)
      return list_filter('sub', 'apache.org', 'members', archivers), File.mtime(LIST_TIME)
    end

    # Return an array of members-notify@ subscribers followed by the file update time
    def self.members_notify_subscribers(archivers=true)
      return list_filter('sub', 'apache.org', 'members-notify', archivers), File.mtime(LIST_TIME)
    end

    # Intended for checking who sees emails on the list
    def self.sub_digest(dom, list, archivers=false)
      subs = list_filter('sub', dom, list, archivers)
      digs = list_filter('digest', dom, list, archivers)
      if subs.nil?
        subs = digs # may also be nil
      elsif !digs.nil? # i.e. neither nil, so merge
        subs = (subs + digs).uniq
      end
      return subs, File.mtime(LIST_TIME)
    end

    # Return an array of private@pmc subscribers (including digests) followed by the file update time
    # Returns nil if the subs file does not exist
    # By default does not return the standard archivers
    # pmc can either be a pmc name, in which case it uses private@<pmc>.apache.org
    # or it can be an ASF list name, e.g. w3c@apache.org
    def self.private_subscribers(pmc, archivers=false)
      return [] unless Dir.exist? LIST_BASE
      parts = pmc.split('@', 3) # want to detect trailing '@'
      if parts.length == 1
        dom = "#{pmc}.apache.org"
        list = 'private'
      elsif parts.length == 2 && %w{apache.org apachecon.com}.include?(parts[1])
        dom = parts[1]
        list = parts[0]
      else
        raise "Unexpected parameter: #{pmc}"
      end
      subs = list_filter('sub', dom, list, archivers)
      digs = list_filter('digest', dom, list, archivers)
      if subs.nil?
        subs = digs # may also be nil
      elsif !digs.nil? # i.e. neither nil, so merge
        subs = (subs + digs).uniq
      end
      return subs, File.mtime(LIST_TIME)
  end

    # Return security subscribers (including digests)
    def self.security_subscribers(pmc, archivers=false)
      return [] unless Dir.exist? LIST_BASE
      subs = list_filter('sub', "#{pmc}.apache.org", 'security', archivers)
      digs = list_filter('digest', "#{pmc}.apache.org", 'security', archivers)
      if subs.nil?
        subs = digs # may also be nil
      elsif !digs.nil? # i.e. neither nil, so merge
        subs = (subs + digs).uniq
      end
      return subs, File.mtime(LIST_TIME)
    end

    # return a hash of subscriptions for the list of emails provided
    # the following keys are added to the response hash:
    # :subtime - the timestamp when the data was last updated
    # :subscriptions - an array of pairs: [list name, subscriber email]
    # N.B. not the same format as the moderates() method
    def self.subscriptions(emails, response = {})

      response[:subscriptions] = []
      response[:subtime] = File.mtime(LIST_TIME)

      emailslc = emails.map {|email| ASF::Mail.to_canonical(email.downcase)}
      list_parse('sub') do |dom, list, subs|
        subs.each do |sub|
          if emailslc.include? ASF::Mail.to_canonical(sub.downcase)
            response[:subscriptions] << ["#{list}@#{dom}", sub]
          end
        end
      end
      response
    end

    # return a hash of digest subscriptions for the list of emails provided
    # the following keys are added to the response hash:
    # :digtime - the timestamp when the data was last updated
    # :digests - an array of pairs: [list name, subscriber email]
    # N.B. not the same format as the moderates() method
    def self.digests(emails, response = {})

      response[:digests] = []
      response[:digtime] = File.mtime(LIST_TIME)

      emailslc = emails.map {|email| ASF::Mail.to_canonical(email.downcase)}
      list_parse('dig') do |dom, list, subs|
        subs.each do |sub|
          if emailslc.include? ASF::Mail.to_canonical(sub.downcase)
            response[:digests] << ["#{list}@#{dom}", sub]
          end
        end
      end
      response
    end

    # return the mailing lists which are moderated by any of the list of emails
    # the following keys are added to the response hash:
    # :modtime - the timestamp when the data was last updated
    # :moderates - a hash. key: list name; entry: array of emails that match a moderator for the list
    # N.B. not the same format as the subscriptions() method
    def self.moderates(user_emails, response = {})

      response[:moderates] = {}
      response[:modtime] = File.mtime(LIST_TIME)
      umails = user_emails.map {|m| ASF::Mail.to_canonical(m.downcase)} # outside loop
      list_parse('mod') do |dom, list, emails|
        matching = emails.select {|m| umails.include? ASF::Mail.to_canonical(m.downcase)}
        response[:moderates]["#{list}@#{dom}"] = matching unless matching.empty?
      end
      response
    end

    # helper function for matching against mod and subs entries
    # does the target mail_domain match the current list?
    def self.matches_list?(mail_domain, dom, list)
      # normal tlp style (now also podlings):
      #/home/apmail/lists/commons.apache.org/dev/mod
      #Apache lists (e.g. some non-PMCs)
      #/home/apmail/lists/apache.org/list/mod
      return "#{mail_domain}.apache.org" == dom ||
              (dom == 'apache.org' &&
              (list == mail_domain || list.start_with?("#{mail_domain}-"))
              ) || "#{list}@#{dom}" == mail_domain # e.g. planners@apachecon.com
    end

    # for a mail domain, extract related lists and their moderators
    # also returns the time when the data was last checked
    # returns: [{dev@a.o=>[email1, email2]}, mod-time]
    # if mail_domain is nil, matches all lists except infra test lists
    def self.list_moderators(mail_domain)

      moderators = {}
      list_parse('mod') do |dom, list, subs|

        # drop infra test lists
        next if list =~ /^infra-[a-z]$/
        next if dom == 'incubator.apache.org' && list =~ /^infra-dev2?$/

        # does the list match our target?
        next unless mail_domain.nil? or matches_list?(mail_domain, dom, list)

        moderators["#{list}@#{dom}"] = subs.sort
      end
      return moderators.to_h, File.mtime(LIST_TIME)
    end

    # for a mail domain, extract related lists and their subscribers (default only the count)
    # also returns the time when the data was last checked
    # N.B. by default includes archivers as subscribers
    # For top-level apache.org lists, the mail_domain is either:
    # - the full list name (e.g. press), or:
    # - the list prefix (e.g. legal)
    # If podling==true, then also check for old-style podling names
    # If list_subs==true, return subscriber emails else sub count
    # If skip_archivers==true, exclude archivers
    # Matches:
    # {mail_domain}.apache.org/*
    # apache.org/{mail_domain}(-.*)? (e.g. press, legal)
    # incubator.apache.org/{mail_domain}-.* (if podling==true)
    # Returns: {list}@{dom}
    def self.list_subscribers(mail_domain, _podling=false, list_subs=false, skip_archivers=false)

      subscribers = {}
      list_parse('sub') do |dom, list, subs|

        # drop infra test lists
        next if list =~ /^infra-[a-z]$/
        next if dom == 'incubator.apache.org' && list =~ /^infra-dev2?$/

        # normal tlp style:
        #/home/apmail/lists/commons.apache.org/dev/mod

        # does the list match our target?
        next unless matches_list?(mail_domain, dom, list)

        if skip_archivers
          subscribers["#{list}@#{dom}"] = list_subs ? subs.reject {|sub| is_archiver?(sub)}.sort : subs.reject {|sub| is_archiver?(sub)}.size
        else
          subscribers["#{list}@#{dom}"] = list_subs ? subs.sort : subs.size
        end
      end
      return subscribers.to_h, File.mtime(LIST_TIME)
    end

    # for a mail domain, extract related lists and their subscribers (default only the count)
    # also returns the time when the data was last checked
    # N.B. excludes known archivers
    # For top-level apache.org lists, the mail_domain is either:
    # - the full list name (e.g. press), or:
    # - the list prefix (e.g. legal)
    # If podling==true, then also check for old-style podling names
    # If list_subs==true, return subscriber emails else sub count
    # Matches:
    # {mail_domain}.apache.org/*
    # apache.org/{mail_domain}(-.*)? (e.g. press, legal)
    # incubator.apache.org/{mail_domain}-.* (if podling==true)
    # Returns: {list}@{dom}
    def self.list_subs(mail_domain, podling=false, list_subs=false)
      self.list_subscribers(mail_domain, podling, list_subs, true)
    end

    # returns the list time
    def self.list_time
      File.mtime(LIST_TIME)
    end

    def self.list_archivers
      list_parse('sub') do |dom, list, subs|
        yield [dom, list, subs.select {|s| is_archiver? s}.map {|m| [m, archiver_type(m, dom, list)].flatten}]
      end
    end

    # return the [domain, list, types=public|private|...] for all entries in the subscriber listings
    # the subscribers are not included
    def self.list_types(show_all=false)
      list_archivers do |dom, list, subs|
        types = {}
        subs.each do |sub|
          type = sub[2]
          types[type] = 1 unless %w(alias direct).include? type
        end
        type = types.keys.sort.join(',')
        yield [dom, list, type] if show_all || type == 'public'
      end
    end

    # return a hash of lists for a project, together with privacy setting
    # tlp - the prefix for the full domain
    # This is a replacement for ASF::Mail.lists
    def self.domain_lists(project, show_all)
      lists = {}
      list_types(show_all) do |dom, list, type|
        if matches_list?(project, dom, list)
          lists["#{list}@#{dom}"] = type
        end
      end
      lists
    end
    # return the [domain, list] for all entries in the subscriber listings
    # the subscribers are not included
    def self.each_list
      Find.find(LIST_CACHE) do |path|
        parts = path.split('/')
        if parts[-1] == 'sub'
          yield [parts[-3], parts[-2]]
        end
      end
    end

    # Parse the marker files: modpost, modsub, remote
    # Return hash: key="domain list", value=hash containing :modsub, :modpost, :remote
    # if the list has the corresponding file
    # BETA: API may change!
    def self.parse_markers
      hash = Hash.new {|h,k| h[k] = Hash.new}
      File.open(File.join(LIST_BASE, 'list-modsub')).each do |line|
        hash[line.chomp][:modsub] = 1
      end
      File.open(File.join(LIST_BASE, 'list-modpost')).each do |line|
        hash[line.chomp][:modpost] = 1
      end
      File.open(File.join(LIST_BASE, 'list-remote')).each do |line|
        hash[line.chomp][:remote] = 1
      end
      hash
    end

    private

    # return the archiver type as array: [:MBOX|:PONY|:MINO|:MAIL_ARCH|:MARKMAIL|:WHIMSY, 'public'|'private'|'alias'|'direct']
    # minotaur archiver names do not include any public/private indication
    def self.archiver_type(email, dom, list)
      case email
        when ARCH_MBOX_PUB then return [:MBOX, 'public']
        when ARCH_MBOX_PRV then return [:MBOX, 'private']
        when ARCH_MBOX_RST then return [:MBOX, 'restricted']
        when ARCH_PONY_PUB then return [:PONY, 'public']
        when ARCH_PONY_PRV then return [:PONY, 'private']
        when ARCH_EXT_MAIL_ARCHIVE then return [:MAIL_ARCH, 'public']
        # normal archiver routed via .qmail-[tlp-]list-archive
        when "#{list}-archive@#{dom}" then return [:MINO, 'alias']
        # Direct mail to minotaur
        when "apmail-#{dom.split('.').first}-#{list}-archive@www.apache.org" then return [:MINO, 'direct']
        # Unexpected archiver email commits-archive@incubator.apache.org for commits.deprecated@incubator.apache.org
        # INFRA-21658
        when "#{list.chomp('.deprecated')}-archive@#{dom}" then return [:MINO, 'alias']
        else
          return [:MARKMAIL, 'public'] if is_markmail_archiver?(email)
          # Whimsy currently only 'archives' private lists
          return [:WHIMSY, 'private'] if is_whimsy_archiver?(email)
      end
      raise "Unexpected archiver email #{email} for #{list}@#{dom}" # Should not happen?
    end

    # Is the email a minotaur archiver?
    def self.is_mino_archiver?(e)
      e =~ /.-archive@([^.]+\.)?(apache\.org|apachecon\.com)$/
    end

    # Is the email a Whimsy archiver?
    def self.is_whimsy_archiver?(e)
      e =~ /@whimsy[-\da-z]*\.apache\.org$/
    end

    # Is the email a markmail archiver?
    def self.is_markmail_archiver?(e)
      e =~ ARCH_EXT_MARKMAIL_RE
    end

    def self.is_archiver?(e)
      ARCHIVERS.include?(e) or is_mino_archiver?(e) or is_whimsy_archiver?(e) or is_markmail_archiver?(e)
    end

    def self.is_private_archiver?(e)
      [ARCH_MBOX_PRV, ARCH_PONY_PRV].include?(e) or
        e =~ /^security-archive@.*\.apache\.org$/ or
        e =~ /^apmail-\w+-security-archive@www.apache.org/ # direct subscription
    end

    def self.downcase(array)
      array.map(&:downcase)
    end

    def self.isRecent(file)
      return File.exist?(file) && (Time.now - File.mtime(file)) < 60 * 60 * 5
    end

    # Filter the appropriate list, matching on domain and list
    # Params:
    # - type: 'mod' or 'sub' or 'digest' or 'allow'
    # - matchdom: must match the domain (e.g. 'httpd.apache.org')
    # - matchlist: must match the list (e.g. 'dev')
    # - archivers: whether to include standard ASF archivers (default true)
    # The email addresses are returned as an array. May be empty.
    # If there is no match, then nil is returned
    def self.list_filter(type, matchdom, matchlist, archivers=true)
      list = File.join(LIST_CACHE, matchdom, matchlist, type)
      if File.exist? list
        if archivers
          return File.read(list).split("\n")
        else
          return File.read(list).split("\n").reject {|e| is_archiver?(e)}
        end
      end
      return nil
    end

    # Scans the cache files
    # Param: type = 'mod' or 'sub' or 'dig'
    # Yields:
    # - domain (e.g. [xxx.].apache.org)
    # - list (e.g. dev)
    # - emails as an array
    def self.list_parse(type, &block)
      case type
      when 'dig'
        type = 'digest'
      when 'sub', 'mod'
        # pass
      else
        raise ArgumentError.new('type: expecting dig, mod or sub')
      end
      ctime = @@file_times[type] || 0
      mtime = File.mtime(LIST_TIME).to_i
      if mtime <= ctime
        cached = @@file_parsed[type]
        if cached
          begin
            cached.each do |d, l, m|
              block.call d, l, m # these are already frozen
            end
            return
          rescue WeakRef::RefError
            @@file_times[type] = 0
          end
        end
      else
        @@file_parsed[type] = nil
      end

      cache = [] # see if this preserves mod cache
      Find.find(LIST_CACHE) do |path|
        parts = path.split('/')
        if parts[-1] == type
          dom = parts[-3].downcase.freeze
          list = parts[-2].downcase.freeze
          begin
            # During mail migration links are used - allow for broken ones
            mails = File.read(path).split("\n")
            cache << [dom, list, mails]
            yield [dom, list, mails]
          rescue StandardError => e
            Wunderbar.warn e
          end
        end
      end
      @@file_parsed[type] = WeakRef.new(cache)
      @@file_times[type] = mtime
      nil # don't return file contents
    end

    # Standard ASF archivers
    ARCH_MBOX_PUB = 'archiver@mbox-vm.apache.org'
    ARCH_MBOX_PRV = 'private@mbox-vm.apache.org'
    ARCH_MBOX_RST = 'restricted@mbox-vm.apache.org'

    ARCH_PONY_PUB = 'archive-asf-public@cust-asf.ponee.io'
    ARCH_PONY_PRV = 'archive-asf-private@cust-asf.ponee.io'

    # Standard external archivers (necessarily public)
    ARCH_EXT_MAIL_ARCHIVE = 'archive@mail-archive.com'
    ARCH_EXT_MARKMAIL_RE = %r{^\w+\.\w+\.\w+@.\.markmail\.org$} # one.two.three@a.markmail.org

    ARCHIVERS = [ARCH_PONY_PRV, ARCH_PONY_PUB,
                 ARCH_MBOX_PUB, ARCH_MBOX_PRV, ARCH_MBOX_RST, ARCH_EXT_MAIL_ARCHIVE]

    # Private archivers
    ARCHIVERS_PRV = [ARCH_PONY_PRV, ARCH_MBOX_PRV]
     # TODO alias archivers: either add list or use RE to filter them

    LIST_BASE = ASF::Config[:subscriptions] # allow overrides for testing etc

    # If this file exists, it is the time when the data was last extracted
    # The mods and subs files are only updated if they have changed
    LIST_TIME = File.join(LIST_BASE, 'list-start')

    LIST_CACHE = File.join(LIST_BASE, 'cache')

  end
end

if __FILE__ == $0
  $LOAD_PATH.unshift '/srv/whimsy/lib'
  require 'whimsy/asf'
  p ASF::MLIST.parse_markers
  exit
  domain = ARGV.shift || 'whimsical'
  mlist = ASF::Committee.find(domain).mail_list
  p mlist
  p ASF::MLIST.domain_lists(mlist, false)
  p ASF::MLIST.domain_lists(mlist, true)
  p ASF::MLIST.list_subscribers(domain)
  p ASF::MLIST.list_subscribers(domain, false, false, true)
  p ASF::MLIST.list_subs(domain)
  p ASF::MLIST.list_subscribers(domain, false, true)
  p ASF::MLIST.list_subscribers(domain, false, true, true)
  p ASF::MLIST.list_subs(domain, false, true)
  p ASF::MLIST.list_moderators(domain, true)
  p ASF::MLIST.private_subscribers(domain)
  p ASF::MLIST.moderates(['chrisd@apache.org'])
  p ASF::MLIST.digests(['chrisd@apache.org'])
end
