yacy_search_server/build.gradle

buildscript {
    dependencies {
        classpath "org.mini2Dx:parcl:1.7.1"
    }
}

plugins {
  id 'java'
  id 'distribution'
  id 'maven-publish'
  id 'com.github.johnrengelman.shadow' version '5.1.0'
}

apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'application'
apply plugin: "org.mini2Dx.parcl"

repositories {
    mavenCentral()
}

sourceSets {
    main {
        java {
	    srcDirs = ['source']
        }
    }

    test {
        java {
            srcDirs = ['test/java']
        }
    }
}

group = 'net.yacy'
version = '1.925'
description = 'YaCy Search Server'
java.sourceCompatibility = JavaVersion.VERSION_1_8
java.targetCompatibility = JavaVersion.VERSION_1_8

mainClassName = "net.yacy.yacy"
applicationDefaultJvmArgs = ["-Xmx1024m"]

// create property store for filterTokens (here, to make sure it exist)
project.ext.filterTokens = new Properties()

parcl {
   exe {
      exeName = "YaCy"
   }

   app {
      vmArgs = ["-Xmx1g"]
      appName = "YaCy"
      icon = "addon/YaCy.icns"
      applicationCategory = "Search-Engine"
      displayName = 'YaCy Search Engine'
      identifier = 'net.yacy'
      copyright = 'Copyright 2021 Michael Peter Christen et al.'
      zipName = 'YaCy.zip'
   }

   linux {
      binName = "YaCy"
   }
}

tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}

dependencies {
//  implementation project ('libbuild:J7Zip-modified') // support library in libbuild (fyi: downside groovy/gradle use internal *.class instead of *.jar = trouble in task packageDist)
//  implementation 'net.yacy.extlib:J7Zip-modified:1.02' // alternatively support library from mavenLocal() repo (use publishToMavenLocal in subproject first) 
    implementation fileTree(dir: 'lib', include: 'J7Zip-modified.jar') // switched back to use local lib/jar for this one library (avoiding sub-project execution w/o need)

    implementation 'org.apache.james:apache-mime4j:0.6'
    implementation 'org.apache.calcite.avatica:avatica-core:1.13.0'
    implementation 'org.bouncycastle:bcmail-jdk15on:1.69'
    implementation 'org.bouncycastle:bcpkix-jdk15on:1.69'
    implementation 'org.bouncycastle:bcprov-jdk15on:1.69'
    implementation 'org.bouncycastle:bcutil-jdk15on:1.69'
    implementation 'org.apache.calcite:calcite-core:1.18.0'
    implementation 'org.apache.calcite:calcite-linq4j:1.18.0'
    implementation 'net.sourceforge.jchardet:jchardet:1.0'            
    implementation 'com.twelvemonkeys.common:common-image:3.3.2'
    implementation 'com.twelvemonkeys.common:common-io:3.3.2'
    implementation 'com.twelvemonkeys.common:common-lang:3.3.2'
    implementation 'commons-codec:commons-codec:1.14'
    implementation 'org.apache.commons:commons-collections4:4.4'
    implementation 'org.apache.commons:commons-compress:1.21'
    implementation 'commons-fileupload:commons-fileupload:1.4'
    implementation 'commons-io:commons-io:2.7'
    implementation 'commons-lang:commons-lang:2.6'
    implementation 'org.apache.commons:commons-lang3:3.12.0'
    implementation 'commons-logging:commons-logging:1.2'
    implementation 'org.apache.commons:commons-math3:3.4.1'
    implementation 'org.apache.pdfbox:fontbox:2.0.15'
    implementation 'com.google.guava:guava:25.1-jre' 
    implementation 'com.hazelcast:hazelcast:4.2'
    implementation 'org.eclipse.jetty.http2:http2-client:9.4.34.v20201102'
    implementation 'org.eclipse.jetty.http2:http2-common:9.4.34.v20201102'
    implementation 'org.eclipse.jetty.http2:http2-http-client-transport:9.4.34.v20201102'
    implementation 'org.apache.httpcomponents:httpclient:4.5.12'
    implementation 'org.apache.httpcomponents:httpcore:4.4.13'
    implementation 'org.apache.httpcomponents:httpmime:4.5.12'
    implementation 'com.ibm.icu:icu4j:63.1'
    implementation 'com.twelvemonkeys.imageio:imageio-bmp:3.3.2'
    implementation 'com.twelvemonkeys.imageio:imageio-core:3.3.2'
    implementation 'com.twelvemonkeys.imageio:imageio-metadata:3.3.2'
    implementation 'com.twelvemonkeys.imageio:imageio-tiff:3.3.2'
    implementation 'com.fasterxml.jackson.core:jackson-annotations:2.11.2'
    implementation 'com.fasterxml.jackson.core:jackson-core:2.11.2'
    implementation 'com.fasterxml.jackson.core:jackson-databind:2.11.2'
    implementation 'oro:oro:2.0.8'
    implementation 'net.jthink:jaudiotagger:2.2.5'
    implementation 'jcifs:jcifs:1.3.17'
    implementation 'org.slf4j:jcl-over-slf4j:1.7.25'
    implementation 'javax.servlet:javax.servlet-api:3.1.0'
    implementation 'org.eclipse.jetty:jetty-client:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-continuation:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-deploy:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-http:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-io:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-jmx:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-proxy:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-security:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-server:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-servlet:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-servlets:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-util:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-webapp:9.4.35.v20201120'
    implementation 'org.eclipse.jetty:jetty-xml:9.4.35.v20201120'
    implementation 'com.jcraft:jsch:0.1.54'
    implementation 'com.googlecode.json-simple:json-simple:1.1.1'
    implementation 'net.arnx:jsonic:1.3.10'
    implementation 'org.jsoup:jsoup:1.14.2'
    implementation 'org.jwat:jwat-archive-common:1.1.1'
    implementation 'org.jwat:jwat-common:1.1.1'
    implementation 'org.jwat:jwat-gzip:1.1.1'
    implementation 'org.jwat:jwat-warc:1.1.1'
    implementation 'com.cybozu.labs:langdetect:1.1-20120112'
    implementation 'org.slf4j:log4j-over-slf4j:1.7.32'
    implementation 'org.slf4j:jcl-over-slf4j:1.7.25'
    implementation 'com.drewnoakes:metadata-extractor:2.11.0'
    implementation 'io.dropwizard.metrics:metrics-core:3.2.2'
    implementation 'io.dropwizard.metrics:metrics-jmx:4.1.5'
    implementation 'io.opentracing:opentracing-api:0.33.0'
    implementation 'io.opentracing:opentracing-noop:0.33.0'
    implementation 'io.opentracing:opentracing-util:0.33.0'
    implementation 'org.apache.pdfbox:pdfbox:2.0.15'
    implementation 'org.apache.poi:poi:3.17'
    implementation 'org.apache.poi:poi-scratchpad:3.17'
    implementation 'org.rrd4j:rrd4j:3.2'
    implementation 'org.slf4j:slf4j-api:1.7.32'
    implementation 'org.slf4j:slf4j-jdk14:1.7.32'
    implementation 'org.apache.solr:solr-core:8.8.1'
    implementation 'org.apache.solr:solr-solrj:8.8.1'
    implementation 'org.locationtech.spatial4j:spatial4j:0.6'
    implementation 'org.codehaus.woodstox:stax2-api:3.1.4'
    implementation 'org.bitlet:weupnp:0.1.4'
    implementation 'org.codehaus.woodstox:woodstox-core-asl:4.4.1'
    implementation 'xml-apis:xml-apis:1.4.01'
    implementation 'com.adobe.xmp:xmpcore:5.1.3'
    implementation 'org.tukaani:xz:1.8'
    implementation 'org.apache.zookeeper:zookeeper:3.4.14'

    testImplementation('junit:junit:4.13.2')
}

// exclude some unused/not needed automatic dependency imports
configurations.implementation {
    exclude group: 'com.google.code.findbugs', module: 'jsr305'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-client'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-java-client'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-java-server'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-server'
}

// output to lib/yacycore.jar (like with ant with downside by determine class-path)
jar {
    archiveFileName = 'yacycore.jar'
    destinationDirectory  = new File('lib')
    manifest {
        attributes(
            "Main-Class": mainClassName ,
            "Class-Path": configurations.runtimeClasspath.collect { it.name }.join(' ')               
        )
    }
}

shadowJar.zip64 = true // saw build error: Execution failed for task ':shadowJar'. shadow.org.apache.tools.zip.Zip64RequiredException: archive contains more than 65535 entries.

// mimic current config with dependencies in lib directory
// @TODO: maybe removed after complete Gradle migration
/**
task copyDependenciesToLib(type: Copy) {
    def destination = project.file("lib")
    into destination
    from configurations.compileClasspath
    eachFile { // prevent overwriting existing files
        if (it.getRelativePath().getFile(destination).exists()) {
            it.exclude()
        }
    }
    build.dependsOn(copyDependenciesToLib)
}
*/

// runs the support tool which creates gitbuildnumber.properties from info of local git repository
task runGitComInf(type: JavaExec) {
    classpath = files('libbuild/GitComInf-all.jar')
    args = [project.projectDir.getPath(), project.projectDir.getPath() + "/build/tmp/gitbuildnumber.properties"]
    mainClass = 'GitComInf'
}

// prepare yacyBuildProperties.java from template (needed after commits or new versions)
// read values from property file generated by GitComInf tool
import org.apache.tools.ant.filters.ReplaceTokens
task prepYaCyProperties (type: Copy, dependsOn : ['runGitComInf']) {
    mustRunAfter runGitComInf
    def propfile = "build/tmp/gitbuildnumber.properties";
    if (project.file(propfile).exists()) { // must check otherwise any action may stop after clean on this error
        file("build/tmp/gitbuildnumber.properties").withReader { 
            Properties props = new Properties()
            props.load(it)
            project.ext.filterTokens.putAll(props)
        }
    } else 
        logger.error("prepYaCyProperties: file " + propfile + " is missing, you should re-run runGitComInf")
    
    String fdirname = sourceSets.main.java.srcDirs[0].name + '/net/yacy/peers/operation'
    copy {
        from fdirname
        include('yacyBuildProperties.java.template')
        rename 'yacyBuildProperties.java.template', 'yacyBuildProperties.java'
        into (fdirname)

        filter(org.apache.tools.ant.filters.ReplaceTokens, tokens: project.ext.filterTokens)
        filter(ReplaceTokens, tokens: [REPL_VERSION: version, REPL_PKGMANAGER:"false", REPL_RESTARTCMD: "/etc/init.d/yacy restart"]) // gradle.project.version + some defaults from ant
    }
}
compileJava.dependsOn(prepYaCyProperties) // must be executed before compile (due to source template filtering)

// implement ant task to simulate old Ant build for htroot,
// to avoid import of complete build.xml (as currently used)
task compileHtrootServlets (dependsOn: ['jar']) {

    ant {
        path(id:'project.class.path') {
            // place yacycore.jar on classpath (not automatically done)
            pathelement (location: jar.archiveFile.get())
            // construct path elements
            configurations.compileClasspath.each { pathelement (location: it) }
        }
        javac (srcdir:'htroot', excludes:'processing/**', 
            source:java.sourceCompatibility, target:java.targetCompatibility,
            debug:"true", debuglevel:"lines,vars,source", includeantruntime:"false", encoding:"UTF-8") {
            classpath (refid:'project.class.path')
        }
    }
}
startScripts.dependsOn(compileHtrootServlets)

// copy additional files into directory used to create distribution archive (distZip/distTar)
task copyFilesToDistDir (dependsOn : ['compileJava', 'jar', 'compileHtrootServlets']) {  
    String instDir = "$buildDir/RELEASE/MAIN/" 
    outputs.files  fileTree(instDir)

        copy {
            from 'addon'
            into instDir + "addon"
        }
        copy {
            from 'bin'
            into instDir + "bin"
        }
        copy {
            from 'defaults'
            into instDir + "defaults"
        }
        copy {
            from 'dictionaries'
            into instDir + "dictionaries"
        }
        copy {
            from 'examples'
            into instDir + "examples"
        }
        copy {
            from 'htroot'
            into instDir + "htroot"
        }
        copy {
            from 'langdetect'
            into instDir + "langdetect"
        }
        copy {
            from 'lib'
            include '*.License'
            into instDir + "lib"
        }
        copy {
            from 'libbuild'
            exclude '**/build/**'
            exclude '**/target/**'
            into instDir + "libbuild"
        }
        copy {
            from 'locales'
            into instDir + "locales"
        }
        copy {
            from 'skins'
            into instDir + "skins"
        }
        copy {
            from 'source'
            into instDir + "source"
        }
        copy {
            from 'vocabularies'
            into instDir + "vocabularies"
        }
        copy {
            from new File('.') 
                include '*.bat' 
                include '*.sh'
                include 'gradlew'
                include 'AUTHORS'
                include 'COPYRIGHT'
                include 'NOTICE'
                include 'readme.txt'
                include 'ChangeLog'
                include 'gpl.txt'
                include 'yacy.yellow'
                include '*.properties'
                include 'build.gradle'
                include 'settings.gradle'
                include 'build.xml'
            into instDir 
        }        
}
distZip.dependsOn(copyFilesToDistDir)
distTar.dependsOn(copyFilesToDistDir)

// declare distribution source directory to be added to archives
distributions {
    main {
        // Gradle appends always the version to the BaseName so this doesn't work, 
        // legacy archive name must be archieved by file rename
        //
        // String Dst = project.ext.filterTokens.get('DSTAMP')
        // String rNr = project.ext.filterTokens.get('releaseNr')
        // String branch = project.ext.filterTokens.get('branch')
        //
        // distributionBaseName = 'yacy' + branch + '_v' + version + '_' + Dst + '_' + rNr

        contents { // fyi: content completed by copyFilesToDistDir
            from 'build/RELEASE/MAIN'
        }
    }
}

// copy runtime dependencies (jar) to the distribution directory
task copyDependenciesForDistribution {
// do a late copy of dependencies to dist lib directory to not interfere with standard distZip
// which includes the lib (and bin) directory by default
// actually if copied early (e.g. task copyFilesToDistDir) final Gradle zip/tar archive has all files double in lib dir (believe it or not)

        String instDir = "$buildDir/RELEASE/MAIN/"
        copy { // needed for legacy distribution file (task packageDist) - (fyi: gradle.distXxx includes it autom. in dist archive)
            def destination = project.file(instDir + "lib")
            from configurations.compileClasspath
            eachFile { // prevent overwriting existing files
                if (it.getRelativePath().getFile(destination).exists()) {
                    it.exclude()
                }
            }
            into instDir + "lib"
        }
        copy { 
            from 'lib'
            include 'yacycore.jar'
            into instDir + "lib"
        }
}

// create a legacy distribution zip archive
task packageDistZip (type : Zip, dependsOn : ['copyDependenciesForDistribution','copyFilesToDistDir','prepYaCyProperties'] ) {

    String Dst = project.ext.filterTokens.get('DSTAMP')
    String rNr = project.ext.filterTokens.get('releaseNr')
    String branch = project.ext.filterTokens.get('branch')    
    String destName = 'yacy' + branch + '_v' + version + '_' + Dst + '_' + rNr + '.' + archiveExtension.get()

    archiveFileName = destName
    destinationDirectory = layout.buildDirectory.dir('distributions')

    from layout.buildDirectory.dir("RELEASE/MAIN")
    into 'yacy'
}

// create a legacy distribution gz/tar archive
task packageDistTar (type : Tar, dependsOn : ['copyDependenciesForDistribution','copyFilesToDistDir','prepYaCyProperties']) {

    compression 'GZIP'
    String Dst = project.ext.filterTokens.get('DSTAMP')
    String rNr = project.ext.filterTokens.get('releaseNr')
    String branch = project.ext.filterTokens.get('branch')    
    String destName = 'yacy' + branch + '_v' + version + '_' + Dst + '_' + rNr + '.tar.gz' // ! Gradle would use ext .tgz (archiveExtension.get()) 

    archiveFileName = destName
    destinationDirectory = layout.buildDirectory.dir('distributions')

    from layout.buildDirectory.dir("RELEASE/MAIN")
    into 'yacy'
}

// wrapper to create legacy distribution archives
task packageDist (dependsOn : ['copyDependenciesForDistribution','packageDistZip','packageDistTar']) {
    // do a late copy of dependencies to dist lib directory to not interfere with standard distZip
    // which includes the lib (and bin) directory by default (task copyDependenciesForDistribution)
    // actually if copied early (e.g. in task copyFilesToDistDir) final Gradle zip/tar archive has all files double in lib dir (believe it or not)
}

// configure Javadoc task
tasks.withType (Javadoc) {
    destinationDir  project.file("javadoc")
    title = "YaCy " + version + " API Documentation"
    failOnError = false
    options.addStringOption("encoding","UTF-8" ) // or any reason to keep the old "iso-8859-1" ?
    classpath = configurations.compileClasspath
    includes = ['**/*.java']
}

// some legacy stuff packaged in separate archive
// archive in build/distributions
task packagePortalsearch {
    ant {
        tar(destfile:'build/distributions/yacy-portalsearch.tar.gz', compression:'gzip', defaultexcludes:'yes', longfile:'fail') {
            tarfileset (dir:'htroot/portalsearch/', prefix:'yacy/portalsearch/') {
                include(name:'**/*')
            }
            tarfileset (dir:'htroot/jquery/', prefix:'yacy/jquery/') {
                include(name:'**/*')
            }
            tarfileset (dir:"htroot/yacy/ui/img-2", prefix:'yacy/ui/img-2/') {
                include(name:"magnify.png")
                include(name:"cancel_round.png")
                include(name:"stop.png")
            }
        }  
    }
}