/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

String sparkMajorVersion = '3.4'
String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")

def sparkProjects = [
    project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}"),
    project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}"),
    project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}"),
]

configure(sparkProjects) {
  configurations {
    all {
      resolutionStrategy {
        force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson214.get()}"
        force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson214.get()}"
        force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson214.get()}"
      }
    }
  }
}

project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
  apply plugin: 'scala'
  apply plugin: 'com.github.alisiikh.scalastyle'

  sourceSets {
    main {
      scala.srcDirs = ['src/main/scala', 'src/main/java']
      java.srcDirs = []
    }
  }

  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    annotationProcessor libs.immutables.value
    compileOnly libs.immutables.value
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(':iceberg-data')
    implementation project(':iceberg-orc')
    implementation project(':iceberg-parquet')
    implementation project(':iceberg-arrow')
    implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
    implementation("org.apache.datasketches:datasketches-java:${libs.versions.datasketches.get()}")

    compileOnly libs.errorprone.annotations
    compileOnly libs.avro.avro
    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark34.get()}") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.apache.arrow'
      exclude group: 'org.apache.parquet'
      // to make sure netty libs only come from project(':iceberg-arrow')
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'org.roaringbitmap'
    }

    compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}"

    implementation libs.parquet.column
    implementation libs.parquet.hadoop

    implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    implementation(libs.arrow.vector) {
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }

    implementation libs.caffeine

    testImplementation(libs.hadoop3.minicluster) {
      exclude group: 'org.apache.avro', module: 'avro'
      // to make sure netty libs only come from project(':iceberg-arrow')
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
    }
    testImplementation project(path: ':iceberg-hive-metastore')
    testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
    testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
      transitive = false
    }
    testImplementation libs.sqlite.jdbc
    testImplementation libs.awaitility
    // runtime dependencies for running REST Catalog based integration test
    testRuntimeOnly libs.jetty.servlet
  }

  test {
    useJUnitPlatform()
  }

  tasks.withType(Test) {
    // Vectorized reads need more memory
    maxHeapSize '3160m'
  }
}

project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") {
  apply plugin: 'java-library'
  apply plugin: 'scala'
  apply plugin: 'com.github.alisiikh.scalastyle'
  apply plugin: 'antlr'

  configurations {
    /*
     The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This
     bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate
     runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with
     the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add
     the runtime dependency back to only the runtime configuration manually.
    */
    implementation {
      extendsFrom = extendsFrom.findAll { it != configurations.antlr }
    }
  }

  dependencies {
    implementation("org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}")
    implementation libs.roaringbitmap

    compileOnly "org.scala-lang:scala-library"
    compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    compileOnly project(':iceberg-api')
    compileOnly project(':iceberg-core')
    compileOnly project(':iceberg-common')
    compileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
    compileOnly("org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark34.get()}") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.apache.arrow'
      exclude group: 'org.apache.parquet'
      // to make sure netty libs only come from project(':iceberg-arrow')
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'org.roaringbitmap'
    }
    compileOnly libs.errorprone.annotations

    testImplementation project(path: ':iceberg-data')
    testImplementation project(path: ':iceberg-parquet')
    testImplementation project(path: ':iceberg-hive-metastore')
    testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
    testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
    testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
      transitive = false
    }
    // runtime dependencies for running REST Catalog based integration test
    testRuntimeOnly libs.jetty.servlet
    testRuntimeOnly libs.sqlite.jdbc

    testImplementation libs.avro.avro
    testImplementation libs.parquet.hadoop
    testImplementation libs.awaitility
    testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}"

    // Required because we remove antlr plugin dependencies from the compile configuration, see note above
    runtimeOnly libs.antlr.runtime
    antlr libs.antlr.antlr4
  }

  test {
    useJUnitPlatform()
  }

  generateGrammarSource {
    maxHeapSize = "64m"
    arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions']
  }
}

project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersion}") {
  apply plugin: 'com.gradleup.shadow'

  tasks.jar.dependsOn tasks.shadowJar

  sourceSets {
    integration {
      java.srcDir "$projectDir/src/integration/java"
      resources.srcDir "$projectDir/src/integration/resources"
    }
  }

  configurations {
    implementation {
      exclude group: 'org.apache.spark'
      // included in Spark
      exclude group: 'org.slf4j'
      exclude group: 'org.apache.commons'
      exclude group: 'commons-pool'
      exclude group: 'commons-codec'
      exclude group: 'org.xerial.snappy'
      exclude group: 'javax.xml.bind'
      exclude group: 'javax.annotation'
      exclude group: 'com.github.luben'
      exclude group: 'com.ibm.icu'
      exclude group: 'org.glassfish'
      exclude group: 'org.abego.treelayout'
      exclude group: 'org.antlr'
      exclude group: 'org.scala-lang'
      exclude group: 'org.scala-lang.modules'
    }
  }

  dependencies {
    api project(':iceberg-api')
    implementation project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
    implementation project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
    implementation project(':iceberg-aws')
    implementation project(':iceberg-azure')
    implementation(project(':iceberg-aliyun')) {
      exclude group: 'edu.umd.cs.findbugs', module: 'findbugs'
      exclude group: 'org.apache.httpcomponents', module: 'httpclient'
      exclude group: 'commons-logging', module: 'commons-logging'
    }
    implementation project(':iceberg-gcp')
    implementation project(':iceberg-bigquery')
    implementation project(':iceberg-hive-metastore')
    implementation(project(':iceberg-nessie')) {
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }
    implementation (project(':iceberg-snowflake')) {
      exclude group: 'net.snowflake' , module: 'snowflake-jdbc'
    }

    integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
    integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark34.get()}"
    integrationImplementation libs.junit.jupiter
    integrationImplementation libs.junit.platform.launcher
    integrationImplementation libs.slf4j.simple
    integrationImplementation libs.assertj.core
    integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
    integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
    integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')

    // runtime dependencies for running Hive Catalog based integration test
    integrationRuntimeOnly project(':iceberg-hive-metastore')
    // runtime dependencies for running REST Catalog based integration test
    integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
    integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
      transitive = false
    }
    integrationRuntimeOnly libs.jetty.servlet
    integrationRuntimeOnly libs.sqlite.jdbc

    // Not allowed on our classpath, only the runtime jar is allowed
    integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
    integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
    integrationCompileOnly project(':iceberg-api')
  }

  shadowJar {
    configurations = [project.configurations.runtimeClasspath]

    zip64 true

    // include the LICENSE and NOTICE files for the shaded Jar
    from(projectDir) {
      include 'LICENSE'
      include 'NOTICE'
    }

    // Relocate dependencies to avoid conflicts
    relocate 'com.google.errorprone', 'org.apache.iceberg.shaded.com.google.errorprone'
    relocate 'com.google.flatbuffers', 'org.apache.iceberg.shaded.com.google.flatbuffers'
    relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml'
    relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes'
    relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework'
    relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro'
    relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded'
    relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer'
    relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet'
    relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded'
    relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc'
    relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift'
    relocate 'org.apache.hc.client5', 'org.apache.iceberg.shaded.org.apache.hc.client5'
    relocate 'org.apache.hc.core5', 'org.apache.iceberg.shaded.org.apache.hc.core5'
    // relocate Arrow and related deps to shade Iceberg specific version
    relocate 'io.netty', 'org.apache.iceberg.shaded.io.netty'
    relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow'
    relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch'
    relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra'
    relocate 'org.roaringbitmap', 'org.apache.iceberg.shaded.org.roaringbitmap'
    relocate 'org.apache.datasketches', 'org.apache.iceberg.shaded.org.apache.datasketches'

    archiveClassifier.set(null)
  }

  task integrationTest(type: Test) {
    useJUnitPlatform()
    description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
    group = "verification"
    jvmArgs += project.property('extraJvmArgs')
    testClassesDirs = sourceSets.integration.output.classesDirs
    classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path)
    inputs.file(shadowJar.archiveFile.get().asFile.path)
  }
  integrationTest.dependsOn shadowJar
  check.dependsOn integrationTest

  jar {
    enabled = false
  }
}

