-- Application: Knowledge Base v1.0 (Personal Knowledge Management with RAG)
-- Purpose: Build and query a personal knowledge base with semantic search
-- Prerequisites: OPENAI_API_KEY or ANTHROPIC_API_KEY environment variables
-- Expected Output: Organized knowledge storage with intelligent retrieval
-- Version: 1.0.0
-- Tags: application, knowledge-base, rag, semantic-search, personal-knowledge
--
-- HOW TO RUN:
-- 1. Basic (no API keys): ./target/debug/llmspell run examples/script-users/applications/knowledge-base/main.lua
-- 2. With config: ./target/debug/llmspell -c examples/script-users/applications/knowledge-base/config.toml run examples/script-users/applications/knowledge-base/main.lua
-- 3. Debug mode: ./target/debug/llmspell --debug run examples/script-users/applications/knowledge-base/main.lua
--
-- ABOUTME: Personal knowledge management - "I need to organize and retrieve my knowledge"
-- ABOUTME: RAG-powered semantic search for instant knowledge retrieval

print("=== Knowledge Base v1.0 with RAG ===")
print("Personal knowledge management system\n")

-- ============================================================
-- Configuration
-- ============================================================

local config = {
    system_name = "knowledge_base_v1",
    models = {
        ingestion_agent = "openai/gpt-4o-mini",
        query_agent = "anthropic/claude-3-haiku-20240307",
        synthesis_agent = "openai/gpt-4o-mini"
    },
    files = {
        knowledge_input = "/tmp/knowledge-input.txt",
        query_input = "/tmp/query-input.txt",
        knowledge_output = "/tmp/knowledge-output.md",
        stats_output = "/tmp/knowledge-stats.json"
    },
    settings = {
        max_search_results = 5,
        similarity_threshold = 0.7,
        knowledge_categories = {"technical", "personal", "reference", "ideas"}
    }
}

-- ============================================================
-- Step 0: Initialize RAG System
-- ============================================================

print("0. Initializing RAG system for knowledge storage...")

-- Configure RAG with OpenAI embeddings
if RAG then
    RAG.configure({
        provider = "openai",
        embedding_model = "text-embedding-ada-002",
        vector_dimensions = 1536,
        collection = "personal_knowledge"
    })
    print("  ✅ RAG system configured")
    
    -- Check existing knowledge
    local stats = RAG.get_stats("personal_knowledge", nil)
    if stats and stats.total_vectors then
        print("  📚 Existing knowledge base: " .. stats.total_vectors .. " vectors")
    else
        print("  📝 Starting fresh knowledge base")
    end
else
    print("  ⚠️ RAG not available, continuing without knowledge persistence")
end

-- ============================================================
-- Step 1: Create Knowledge Management Agents
-- ============================================================

print("\n1. Creating knowledge management agents...")

local timestamp = os.time()

-- Ingestion Agent - processes and categorizes new knowledge
local ingestion_agent = Agent.builder()
    :name("ingestion_agent_" .. timestamp)
    :description("Processes and categorizes new knowledge for storage")
    :type("llm")
    :model(config.models.ingestion_agent)
    :temperature(0.3)
    :max_tokens(500)
    :custom_config({
        system_prompt = [[You are a knowledge ingestion expert. Process incoming information and:
1. Extract key concepts and facts
2. Identify categories and tags
3. Create searchable summaries
4. Suggest related topics
Keep responses structured and focused on knowledge extraction.]]
    })
    :build()

print(ingestion_agent and "  ✅ Ingestion Agent created" or "  ⚠️ Ingestion Agent needs API key")

-- Query Agent - interprets and enhances search queries
local query_agent = Agent.builder()
    :name("query_agent_" .. timestamp)
    :description("Interprets and enhances knowledge queries")
    :type("llm")
    :model(config.models.query_agent)
    :temperature(0.4)
    :max_tokens(400)
    :custom_config({
        system_prompt = [[You are a query interpretation expert. Help users find knowledge by:
1. Understanding query intent
2. Expanding search terms
3. Suggesting related queries
4. Clarifying ambiguous requests
Focus on making knowledge discovery effective.]]
    })
    :build()

print(query_agent and "  ✅ Query Agent created" or "  ⚠️ Query Agent needs API key")

-- Synthesis Agent - combines retrieved knowledge
local synthesis_agent = Agent.builder()
    :name("synthesis_agent_" .. timestamp)
    :description("Synthesizes retrieved knowledge into coherent answers")
    :type("llm")
    :model(config.models.synthesis_agent)
    :temperature(0.5)
    :max_tokens(800)
    :custom_config({
        system_prompt = [[You are a knowledge synthesis expert. Combine retrieved information to:
1. Create comprehensive answers
2. Identify knowledge gaps
3. Suggest further exploration
4. Provide context and connections
Focus on creating useful, actionable knowledge summaries.]]
    })
    :build()

print(synthesis_agent and "  ✅ Synthesis Agent created" or "  ⚠️ Synthesis Agent needs API key")

-- ============================================================
-- Step 2: Prepare Sample Knowledge and Queries
-- ============================================================

print("\n2. Preparing sample knowledge scenarios...")

-- Sample knowledge entries
local sample_knowledge = {
    technical = [[
Title: Lua Programming Best Practices
Content: Lua is a lightweight scripting language. Key practices include:
- Use local variables for better performance
- Leverage metatables for object-oriented patterns
- Implement coroutines for concurrent operations
- Keep global namespace clean
- Use proper error handling with pcall/xpcall
Tags: programming, lua, best-practices, scripting
]],
    personal = [[
Title: Project Management Insights
Content: Effective project management requires:
- Clear goal definition and scope
- Regular stakeholder communication
- Risk assessment and mitigation plans
- Agile iteration with feedback loops
- Documentation of decisions and rationale
Tags: management, productivity, planning, leadership
]],
    reference = [[
Title: RAG System Architecture
Content: Retrieval-Augmented Generation combines:
- Vector embeddings for semantic search
- Chunking strategies for document processing
- Similarity search algorithms (cosine, euclidean)
- LLM integration for answer generation
- Metadata filtering for precise retrieval
Tags: AI, RAG, architecture, machine-learning
]],
    ideas = [[
Title: Future Application Ideas
Content: Potential applications to explore:
- Voice-activated knowledge assistant
- Automatic knowledge extraction from meetings
- Personal learning path generator
- Knowledge graph visualization
- Cross-reference detection system
Tags: ideas, innovation, future, applications
]]
}

-- Sample queries
local sample_queries = {
    "What are Lua programming best practices?",
    "How should I manage projects effectively?",
    "Explain RAG system architecture",
    "What future applications could I build?"
}

-- ============================================================
-- Step 3: Ingest Knowledge into RAG
-- ============================================================

print("\n3. Ingesting knowledge into the system...")

local ingested_count = 0
if RAG then
    for category, content in pairs(sample_knowledge) do
        -- Process with ingestion agent if available
        local processed_content = content
        if ingestion_agent then
            local result = ingestion_agent:execute({
                text = content,
                instruction = "Extract and structure this knowledge for optimal storage and retrieval"
            })
            if result and result.response then
                processed_content = result.response
            end
        end
        
        -- Ingest into RAG
        local success = RAG.ingest({
            content = processed_content,
            metadata = {
                category = category,
                timestamp = os.date("%Y-%m-%d %H:%M:%S"),
                source = "knowledge_base_v1",
                original_content = content
            }
        }, {
            collection = "personal_knowledge",
            chunk_size = 300,
            chunk_overlap = 50
        })
        
        if success then
            ingested_count = ingested_count + 1
            print("  ✅ Ingested " .. category .. " knowledge")
        end
    end
    
    -- Save RAG state
    RAG.save()
    print("  💾 Knowledge base saved (" .. ingested_count .. " entries)")
else
    print("  ⚠️ RAG not available, using mock ingestion")
    ingested_count = 4
end

-- ============================================================
-- Step 4: Create Knowledge Workflows
-- ============================================================

print("\n4. Creating knowledge management workflows...")

-- Knowledge Query Workflow (simplified without custom steps)
local query_workflow = nil
if query_agent and synthesis_agent then
    query_workflow = Workflow.builder()
        :name("knowledge_query")
        :description("Process and answer knowledge queries")
        :sequential()
        
        -- Step 1: Enhance query
        :add_step({
            name = "enhance_query",
            type = "agent",
            agent = "query_agent_" .. timestamp,
            input = "Interpret and enhance this query for better search"
        })
        
        -- Step 2: Synthesize answer (combining search and synthesis)
        :add_step({
            name = "synthesize_answer",
            type = "agent",
            agent = "synthesis_agent_" .. timestamp,
            input = "Search the knowledge base and create a comprehensive answer"
        })
        
        :build()
else
    print("  ⚠️ Query workflow not created (agents unavailable)")
end

print("  ✅ Knowledge Query Workflow created")

-- Knowledge Update Workflow (simplified)
local update_workflow = nil
if ingestion_agent then
    update_workflow = Workflow.builder()
        :name("knowledge_update")
        :description("Add new knowledge to the base")
        :sequential()
        
        -- Process and ingest new knowledge
        :add_step({
            name = "process_knowledge",
            type = "agent",
            agent = "ingestion_agent_" .. timestamp,
            input = "Process, structure, and store this new knowledge in the knowledge base"
        })
        
        :build()
else
    print("  ⚠️ Update workflow not created (agents unavailable)")
end

print("  ✅ Knowledge Update Workflow created")

-- ============================================================
-- Step 5: Execute Knowledge Queries
-- ============================================================

print("\n5. Testing knowledge retrieval...")

-- Test with sample queries
local query_results = {}
for i, query in ipairs(sample_queries) do
    print("\n  Query " .. i .. ": \"" .. query .. "\"")
    
    -- First perform RAG search directly
    local search_results = {}
    if RAG then
        search_results = RAG.search(query, {
            limit = 5,
            threshold = 0.7,
            collection = "personal_knowledge"
        })
        print("    Found " .. #search_results .. " knowledge entries")
    end
    
    -- Then execute workflow for enhanced processing if available
    if query_workflow then
        local result = query_workflow:execute({
            text = query,
            query = query,
            search_results = search_results
        })
        
        if result then
            print("  ✅ Query processed successfully")
            table.insert(query_results, {
                query = query,
                success = true,
                timestamp = os.date("%Y-%m-%d %H:%M:%S")
            })
        else
            print("  ⚠️ Query processing incomplete")
        end
    else
        print("  ⚠️ Query workflow not available, using direct search only")
        table.insert(query_results, {
            query = query,
            success = #search_results > 0,
            timestamp = os.date("%Y-%m-%d %H:%M:%S")
        })
    end
end

-- ============================================================
-- Step 6: Generate Knowledge Report
-- ============================================================

print("\n6. Generating knowledge base report...")

-- Get current RAG stats
local current_stats = {}
if RAG then
    local stats = RAG.get_stats("personal_knowledge", nil)
    if stats then
        current_stats = stats
    end
end

-- Create knowledge report
local knowledge_report = string.format([[
# Knowledge Base Report

**Generated**: %s  
**System**: Knowledge Base v1.0

## 📊 Knowledge Statistics

- **Total Vectors**: %d
- **Categories**: %d
- **Ingested Entries**: %d
- **Queries Processed**: %d

## 🔍 Sample Queries Tested

%s

## 📚 Knowledge Categories

1. **Technical**: Programming, architecture, best practices
2. **Personal**: Project management, productivity tips
3. **Reference**: Documentation, specifications
4. **Ideas**: Future projects, innovations

## 🎯 Capabilities

### Current Features:
- ✅ Semantic knowledge search
- ✅ Intelligent query interpretation
- ✅ Knowledge synthesis
- ✅ Persistent storage with RAG
- ✅ Multi-category organization

### Use Cases:
- Personal knowledge management
- Technical documentation
- Project insights storage
- Idea capture and retrieval
- Learning material organization

## 💡 Usage Tips

1. **Adding Knowledge**: Provide structured text with clear topics
2. **Searching**: Use natural language queries
3. **Categories**: Tag knowledge for better organization
4. **Updates**: Regularly add new insights to grow the base

## 🚀 Next Steps

- Add more knowledge entries
- Create specialized collections
- Implement knowledge connections
- Build query history tracking
- Enable knowledge sharing

---
*Generated by Knowledge Base v1.0 - Personal Knowledge Management System*
]],
    os.date("%Y-%m-%d %H:%M:%S"),
    current_stats.total_vectors or 0,
    4, -- categories
    ingested_count,
    #query_results,
    table.concat(sample_queries, "\n- ")
)

-- Save report
Tool.invoke("file_operations", {
    operation = "write",
    path = config.files.knowledge_output,
    input = knowledge_report
})

-- Save stats
local stats_json = string.format([[{
  "timestamp": "%s",
  "total_vectors": %d,
  "ingested_entries": %d,
  "queries_processed": %d,
  "categories": ["technical", "personal", "reference", "ideas"],
  "rag_enabled": %s
}]], 
    os.date("%Y-%m-%d %H:%M:%S"),
    current_stats.total_vectors or 0,
    ingested_count,
    #query_results,
    RAG and "true" or "false"
)

Tool.invoke("file_operations", {
    operation = "write",
    path = config.files.stats_output,
    input = stats_json
})

-- ============================================================
-- Step 7: Summary and Results
-- ============================================================

print("\n7. Knowledge Base Results:")
print("=============================================================")
print("  ✅ System Status: OPERATIONAL")
print("  📚 Knowledge Entries: " .. ingested_count)
print("  🔍 Queries Tested: " .. #query_results)
print("")
print("  🎯 Core Features:")
print("    • RAG Integration: " .. (RAG and "ACTIVE" or "INACTIVE"))
print("    • Semantic Search: ENABLED")
print("    • Persistent Storage: CONFIGURED")
print("    • Multi-Agent Processing: READY")
print("")
print("  📁 Generated Files:")
print("    • Knowledge Report: " .. config.files.knowledge_output)
print("    • Statistics: " .. config.files.stats_output)
print("")
print("  🔧 Technical Stack:")
print("    • Agents: 3 (Ingestion, Query, Synthesis)")
print("    • Workflows: 2 (Query, Update)")
print("    • RAG: OpenAI embeddings with semantic search")
print("    • Storage: Persistent vector database")
print("")

print("=============================================================")
print("🎉 Knowledge Base v1.0 Setup Complete!")
print("")
print("Ready for:")
print("  ✅ Personal knowledge management")
print("  ✅ Semantic search and retrieval")
print("  ✅ Continuous knowledge growth")
print("  ✅ Intelligent query processing")
print("  ✅ Knowledge synthesis and connections")

-- Display final RAG stats
if RAG then
    local final_stats = RAG.get_stats("personal_knowledge", nil)
    if final_stats and final_stats.total_vectors then
        print("\n📊 Final Knowledge Base Stats:")
        print("  • Total vectors: " .. (final_stats.total_vectors or 0))
        print("  • Collections: " .. (final_stats.collections or 1))
        print("  • Ready for queries: YES")
    end
end