yacy_search_server/build.gradle

/** for possible future use or parcl (of native apps) - 2022-02-09 (see other commented out sections below)
buildscript {
    dependencies {
        classpath "org.mini2Dx:parcl:1.8.0"
    }
}
*/

plugins {
    id 'java'
    id 'distribution'
    id 'maven-publish'
    id 'com.github.johnrengelman.shadow' version '7.1.2' // uberJar plugin
    id 'com.github.langmo.gradlensis' version '0.1.0'  // NSIS plugin
}

apply plugin: 'java'
apply plugin: 'eclipse'
apply plugin: 'application'
/** apply plugin: "org.mini2Dx.parcl" // for possible future use parcl - 2022-02-09 */

repositories {
    mavenCentral()
}

sourceSets {
    main {
        java {
	    srcDirs = ['source']
        }
    }

    test {
        java {
            srcDirs = ['test/java']
        }
    }

    htroot { // optional sourceset to have htroot servelets available/visible in development IDE
        java {
	    srcDirs = ['htroot']
            exclude 'processing/**'
            compileClasspath = configurations.compileClasspath + jar.outputs.files
        }
    }

}

group = 'net.yacy'
version = '1.925'
description = 'YaCy Search Server'
java.sourceCompatibility = JavaVersion.VERSION_1_8
java.targetCompatibility = JavaVersion.VERSION_1_8

mainClassName = "net.yacy.yacy"
// applicationDefaultJvmArgs = ["-Xmx1024m"] // // limit memory use (below gradle default)

// create property store for filterTokens (here, to make sure it exist)
project.ext.filterTokens = new Properties()

/** for possible future use of parcl - 2022-02-09
parcl {
   exe {
      exeName = "YaCy"
   }

   app {
      vmArgs = ["-Xmx1g"]
      appName = "YaCy"
      icon = "addon/YaCy.icns"
      applicationCategory = "Search-Engine"
      displayName = 'YaCy Search Engine'
      identifier = 'net.yacy'
      copyright = 'Copyright 2021 Michael Peter Christen et al.'
      zipName = 'YaCy.zip'
   }

   linux {
      binName = "YaCy"
   }
}
*/

tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}

dependencies {
// used by YaCy code
//  implementation project ('libbuild:J7Zip-modified') // support library in libbuild (fyi: downside groovy/gradle use internal *.class instead of *.jar = trouble in task packageDist)
//  implementation 'net.yacy.extlib:J7Zip-modified:1.0.2' // alternatively support library from mavenLocal() repo (use publishToMavenLocal in subproject first) 
    implementation fileTree(dir: 'lib', include: 'J7Zip-modified-1.0.2.jar') // switched back to use local lib/jar for this one library (avoiding sub-project execution w/o need)
    implementation group: 'org.bouncycastle',   name: 'bcmail-jdk15on',     version: '1.69'
    implementation group: 'org.apache.commons', name: 'commons-compress',   version: '1.21'
    implementation group: 'commons-fileupload', name: 'commons-fileupload', version: '1.4'
    implementation group: 'com.google.guava',   name: 'guava',              version: '31.0.1-jre' 
    implementation group: 'com.hazelcast',      name: 'hazelcast',          version: '4.2.4'
    implementation group: 'com.ibm.icu',        name: 'icu4j',              version: '63.1'
    implementation group: 'com.twelvemonkeys.imageio', name: 'imageio-bmp', version: '3.8.1'
    implementation group: 'com.twelvemonkeys.imageio', name: 'imageio-tiff',version: '3.8.1'
    implementation group: 'net.jthink',         name: 'jaudiotagger',       version: '2.2.5'
    implementation group: 'javax.servlet',      name: 'javax.servlet-api',  version: '3.1.0'
    implementation group: 'net.sourceforge.jchardet',  name: 'jchardet',    version: '1.0' 
    implementation group: 'jcifs',              name: 'jcifs',              version: '1.3.17'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-http',         version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-proxy',        version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-security',     version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-server',       version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-servlets',     version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-util',         version: '9.4.35.v20201120'
    implementation group: 'org.eclipse.jetty',  name: 'jetty-webapp',       version: '9.4.35.v20201120'
    implementation group: 'com.jcraft',         name: 'jsch',               version: '0.1.54'
    implementation group: 'com.googlecode.json-simple',name: 'json-simple', version: '1.1.1'
    implementation group: 'org.jsoup',          name: 'jsoup',              version: '1.14.3'
    implementation group: 'org.jwat',           name: 'jwat-warc',          version: '1.1.1'
    implementation group: 'com.cybozu.labs',    name: 'langdetect',         version: '1.1-20120112'
    implementation group: 'com.drewnoakes',     name: 'metadata-extractor', version: '2.11.0'
    implementation group: 'org.apache.pdfbox',  name: 'pdfbox',             version: '2.0.25'
    implementation group: 'org.apache.poi',     name: 'poi',                version: '3.17'
    implementation group: 'org.apache.poi',     name: 'poi-scratchpad',     version: '3.17'
    implementation group: 'org.apache.solr',    name: 'solr-core',          version: '8.8.1'
    implementation group: 'org.apache.solr',    name: 'solr-solrj',         version: '8.8.1'
    implementation group: 'org.bitlet',         name: 'weupnp',             version: '0.1.4'

    // overwrite version for transient dependencies
    implementation 'org.eclipse.jetty:jetty-client:9.4.35.v20201120'  // for solrj
    implementation 'org.tukaani:xz:1.9' // commons.compress.compressors.xz uses this, but declares it as optional dependency

    testImplementation 'junit:junit:4.13.2'
}

// exclude some unused/not needed automatic dependency imports
configurations.implementation {
    exclude group: 'com.google.code.findbugs', module: 'jsr305'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-client'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-java-client'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-java-server'
    exclude group: 'org.eclipse.jetty',        module: 'jetty-alpn-server'
    // warning 'Class path contains multiple SLF4J bindings' 
    exclude group: 'org.slf4j',                module: 'slf4j-log4j12'
}

// configure jar
jar {
    archiveFileName = 'yacycore.jar'
    manifest {
        attributes(
            "Main-Class": mainClassName ,
            "Class-Path": configurations.runtimeClasspath.collect { it.name }.join(' ')               
        )
    }
}

shadowJar.zip64 = true // saw build error: Execution failed for task ':shadowJar'. shadow.org.apache.tools.zip.Zip64RequiredException: archive contains more than 65535 entries.

// prepare yacyBuildProperties.java from template (needed after commits or new versions)
// read values from property file generated by storeGitComInf tool
import org.apache.tools.ant.filters.ReplaceTokens
task prepYaCyProperties (type: Copy) {
    inputs.files('gitbuildnumber.properties', sourceSets.main.java.srcDirs[0].name + "/net/yacy/peers/operation/yacyBuildProperties.java.template")

    def propfile = "gitbuildnumber.properties";
    if (project.file(propfile).exists()) { // must check otherwise any action may stop after clean on this error
        file("gitbuildnumber.properties").withReader { 
            Properties props = new Properties()
            props.load(it)
            project.ext.filterTokens.putAll(props)
        }
    } else { // on missing properties file use generic data  
        logger.error("prepYaCyProperties: file " + propfile + " is missing, you should re-run storeGitComInf")
        Properties props = new Properties()
        String Dstr = new Date().format("yyyyMMDD")
        String Tstr = new Date().format("yyyyMMDD-HHmmss")
        props.put("branch","-dev")
        props.put ("REPL_DATE",Dstr)
        props.put ("REPL_REVISION_NR",Tstr)
        project.ext.filterTokens.putAll(props)
    }
    
    String fdirname = sourceSets.main.java.srcDirs[0].name + '/net/yacy/peers/operation'
    from fdirname
    include('yacyBuildProperties.java.template')
    rename 'yacyBuildProperties.java.template', 'yacyBuildProperties.java'
    filter(org.apache.tools.ant.filters.ReplaceTokens, tokens: project.ext.filterTokens)
    filter(ReplaceTokens, tokens: [REPL_VERSION: project.version, REPL_PKGMANAGER:"false", REPL_RESTARTCMD: "/etc/init.d/yacy restart"]) // gradle.project.version + some defaults from ant
    into (fdirname)
    
}
compileJava.dependsOn(prepYaCyProperties) // must be executed before compile (due to source template filtering)

// implement ant task to simulate old Ant build for htroot,
// to avoid import of complete build.xml (as currently used)
task compileHtrootServlets (dependsOn: ['jar']) {
    inputs.files(fileTree('htroot') { // allow lifecycle UP-TO-DATE even with doLast block
        include '**/*.java'
        include '**/*.class'  // to detect missing class file
        exclude 'processing/**'
        })
    outputs.files(fileTree('htroot') {
        include '**/*.class'
        })

    doLast{
        if (file(jar.archiveFile).exists()) { // prevent buildscript failing on prime build with fresh git clone due to javac error during gradle init
            // logger.warn("if java files changed in source AND htroot at the same time and build fails during 'Configure project' with javac error -> delete build/libs/yacycore.jar or build directory once")
            ant {
                path(id:'project.class.path') {
                    // place yacycore.jar on classpath (not automatically done)
                    pathelement (location: jar.archiveFile.get())
                    // construct path elements
                    configurations.compileClasspath.each { pathelement (location: it) }
                }
                javac (srcdir:'htroot', excludes:'processing/**',
                    source:java.sourceCompatibility, target:java.targetCompatibility,
                    debug:"true", debuglevel:"lines,vars,source", includeantruntime:"false", encoding:"UTF-8") {
                    classpath (refid:'project.class.path')
                }
            }
        }
    }
}

// copy additional files into directory used to create distribution archive (distZip/distTar)
task copyFilesToDistDir (type: Copy, dependsOn : ['jar', 'compileHtrootServlets']) {  
    String instDir = "$buildDir/RELEASE/MAIN/" 

    from projectDir
    include 'addon/**'
    include 'bin/**'
    include 'defaults/**'
    include 'examples/**'
    include 'gradle/**'
    include 'htroot/**'
    include 'langdetect/**'
    // include 'lib/**'  // do not copy old ant lib, gradle dependencies will be copied into lib by task copy copyDependenciesForDistribution
    include 'libbuild/**'
    include 'locales/**'
    include 'skins/**'
    include 'source/**'
//    include 'vocabularies/**' // not included in prev releases

    include '*.bat' 
    include '*.sh'
    include 'gradlew'
    include 'AUTHORS'
    include 'COPYRIGHT'
    include 'NOTICE'
    include 'readme.txt'
    include 'ChangeLog'
    include 'gpl.txt'
    include 'yacy.yellow'
    include '*.properties'
    include 'build.gradle'
    include 'settings.gradle'
    include 'build.xml'

    exclude '**/build/**'
    exclude '**/target/**'

    into instDir
}

distZip.dependsOn(copyFilesToDistDir)
distTar.dependsOn(copyFilesToDistDir)

// declare distribution source directory to be added to archives
distributions {
    main {
        // Gradle appends always the version to the BaseName so this doesn't work, 
        // legacy archive name must be archieved by file rename
        //
        // String Dst = project.ext.filterTokens.get('REPL_DATE')
        // String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
        // String branch = project.ext.filterTokens.get('branch')
        //
        // distributionBaseName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr
        
        contents { // fyi: content completed by copyFilesToDistDir
            from "${buildDir}/RELEASE/MAIN"
            exclude "lib/**" // lib dir is only for packageDist created
            // define unix/linux file permission
            eachFile { file ->
                if(file.getName().endsWith(".sh")) {
                    file.setMode(0755)
                }
            }
        }
    }
}

// configure distTar archive name
tasks.findByName('assembleDist').dependsOn('prepYaCyProperties') // make sure poperties are loaded before distTar/distZip
distTar {
    compression 'GZIP'
    archiveExtension = 'tar.gz'
    String Dst = project.ext.filterTokens.get('REPL_DATE')
    String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
    String branch = project.ext.filterTokens.get('branch')
    archiveVersion = '' // important as otherwise version is added automatically again to the end of archiveBaseName
    archiveBaseName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr // important to use project.version otherwise depreciated distZip.version is used (wich is now '')
}

// configure distZip archive name
distZip {
    String Dst = project.ext.filterTokens.get('REPL_DATE')
    String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
    String branch = project.ext.filterTokens.get('branch')
    archiveVersion = '' // important as otherwise version is added automatically again to the end of archiveBaseName
    archiveBaseName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr // important to use project.version otherwise depreciated distZip.version is used (wich is now '')
}

// copy runtime dependencies (jar) to the distribution directory
task copyDependenciesForDistribution (type: Copy) {
    // do a late copy of dependencies to dist lib directory to not interfere with standard distZip
    // which includes the lib (and bin) directory by default
    // actually if copied early (e.g. task copyFilesToDistDir) final Gradle zip/tar archive has all files double in lib dir (believe it or not)

    String instDir = "$buildDir/RELEASE/MAIN/"
    // needed for legacy distribution file (task packageDist) - (fyi: gradle.distZip/Tar includes it autom. in dist archive)
    from configurations.runtimeClasspath
    from jar.outputs.files  // add own jar (not included automatically)
    into instDir + "lib" 
}

// create a legacy distribution zip archive
task packageDistZip (type : Zip, dependsOn : ['copyDependenciesForDistribution','copyFilesToDistDir','prepYaCyProperties'] ) {

    String Dst = project.ext.filterTokens.get('REPL_DATE')
    String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
    String branch = project.ext.filterTokens.get('branch')    
    String destName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr + '.' + archiveExtension.get()

    archiveFileName = destName
    destinationDirectory = layout.buildDirectory.dir('distributions/legacyDistFiles')

    from layout.buildDirectory.dir("RELEASE/MAIN")
    into 'yacy'
}

// create a legacy distribution gz/tar archive
task packageDistTar (type : Tar, dependsOn : ['copyDependenciesForDistribution','copyFilesToDistDir','prepYaCyProperties']) {

    compression 'GZIP'
    String Dst = project.ext.filterTokens.get('REPL_DATE')
    String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
    String branch = project.ext.filterTokens.get('branch')    
    String destName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr + '.tar.gz' // ! Gradle would use ext .tgz (archiveExtension.get()) 

    archiveFileName = destName
    destinationDirectory = layout.buildDirectory.dir('distributions/legacyDistFiles')
    from layout.buildDirectory.dir("RELEASE/MAIN") 
    fileMode 0644
    eachFile { file ->
        if(file.getName().endsWith(".sh")) {
            file.setMode(0755)
        }
    }
    into 'yacy'
}

// wrapper to create legacy distribution archives
task packageDist (dependsOn : ['packageDistZip','packageDistTar'],group:'distribution') {
    description 'Create legacy distribution tar.gz and zip archives'
    // do a late copy of dependencies to dist lib directory to not interfere with standard distZip
    // which includes the lib (and bin) directory by default (task copyDependenciesForDistribution)
    // actually if copied early (e.g. in task copyFilesToDistDir) final Gradle zip/tar archive has all files double in lib dir (believe it or not)
}

// configure Javadoc task
tasks.withType (Javadoc) {
    destinationDir  project.file("javadoc")
    title = "YaCy " + project.version + " API Documentation"
    failOnError = false
    options.addStringOption("encoding","UTF-8" ) // or any reason to keep the old "iso-8859-1" ?
    classpath = configurations.compileClasspath
    includes = ['**/*.java']
}

// some legacy stuff packaged in separate archive
// archive in build/distributions
task packagePortalsearch (group:'distribution') {
    description 'Bundles the javascript portalsearch as *.tar.gz distribution'
    ant {
        tar(destfile:"${buildDir}/distributions/yacy-portalsearch.tar.gz", compression:'gzip', defaultexcludes:'yes', longfile:'fail') {
            tarfileset (dir:'htroot/portalsearch/', prefix:'yacy/portalsearch/') {
                include(name:'**/*')
            }
            tarfileset (dir:'htroot/jquery/', prefix:'yacy/jquery/') {
                include(name:'**/*')
            }
            tarfileset (dir:"htroot/yacy/ui/img-2", prefix:'yacy/ui/img-2/') {
                include(name:"magnify.png")
                include(name:"cancel_round.png")
                include(name:"stop.png")
            }
        }  
    }
}

// Configuration of gradle-nsis plugin.
nsis  {
    // path to the NSIS configuration file
    configuration = file("${buildDir}/RELEASE/WINDOWS/build.nsi")
    // folder where the NSIS installer generator is run. All paths 
    // in the NSIS configuration file are relative to this folder.
    runIn =  file("${buildDir}")
}

// prepare the NSIS build script 
task prepNsis (type: Copy, dependsOn: ['prepYaCyProperties']) {
    from projectDir
    include 'build.nsi'
    include 'gpl.txt'
    into "${buildDir}/RELEASE/WINDOWS"

    filter(org.apache.tools.ant.filters.ReplaceTokens, tokens: project.ext.filterTokens)
    filter(ReplaceTokens, tokens: [REPL_VERSION: project.version])

    copy {
        from projectDir
        include 'gpl.txt'
        into buildDir
    }
}

// wrapper to run prepNsis task before nsis createInstaller, to copy some config files
task distWinInstaller (dependsOn:['copyFilesToDistDir','copyDependenciesForDistribution','prepNsis','createInstaller'], group:'distribution') {
    description 'Creates a Win installer distribution exe file'
    tasks.findByName('createInstaller').mustRunAfter 'prepNsis'
}

// runs the support tool which creates gitbuildnumber.properties from info of local git repository
task storeGitComInf(type: JavaExec) {
    outputs.file "gitbuildnumber.properties"
    mustRunAfter(build)
    if (file(".git").exists()) { // check if this is a git repos
        inputs.files ( fileTree(".git") {
            include "refs/**/*"  // these two should work and change after commit
            include "index"
        } )
    }
    classpath = files('libbuild/GitComInf-1.0-all.jar')
    args = [project.projectDir.getPath(), project.projectDir.getPath() + "/gitbuildnumber.properties"]
    mainClass = 'GitComInf'
}

//--- build aliases : define a synonym here if you want a shortcut to run multiple targets
// workaround to run storeGitComInf after a build without adding arbitray dependencies 
// this makes sure gitbuildnumber.properties is availabel and up-to-date
def buildAliases = [
   'build' : ['build','storeGitComInf']
    ]
def expandedTaskList = []
gradle.startParameter.taskNames.each {
    expandedTaskList << (buildAliases[it] ? buildAliases[it] : it)
}
gradle.startParameter.taskNames = expandedTaskList.flatten()

// create a Mac distribution archive
import org.apache.tools.ant.taskdefs.condition.Os
task distMacApp (type: Copy, dependsOn: ['copyFilesToDistDir','copyDependenciesForDistribution'], group: 'distribution') {
    description 'Bundles the project as a Mac distribution - task must run on Mac'
 
    from "${buildDir}/RELEASE/MAIN"  // remember var substitution requires double quote
    into "${buildDir}/RELEASE/MAC/YaCy.app/Contents/MacOS"

    doLast {   
        if (Os.isFamily(Os.FAMILY_MAC))  {
            eachFile { file ->
                if(file.getName().endsWith(".sh")) {
                    file.setMode(0755)
                }
            }
            copy {
                from 'addon/YaCy.app'
                into "${buildDir}/RELEASE/MAC/YaCy.app"
            }
            String Dst = project.ext.filterTokens.get('REPL_DATE')
            String rNr = project.ext.filterTokens.get('REPL_REVISION_NR')
            String branch = project.ext.filterTokens.get('branch')    
            String theArchiveName = 'yacy' + branch + '_v' + project.version + '_' + Dst + '_' + rNr + '.dmg'
            logger.info("task 'distMacApp' execute: " + "'hdiutil create -srcfolder ${buildDir}/RELEASE/MAC/YaCy.app ${buildDir}/distributions/" + theArchiveName + "'")
            ant {
                exec (executable:'hdiutil') {
                    arg (line: "create -srcfolder ${buildDir}/RELEASE/MAC/YaCy.app ${buildDir}/distributions/" + theArchiveName)
                }
            }
        } else {
            logger.error ("this task 'distMacApp' can only run on a Mac")
            delete "${buildDir}/RELEASE/MAC"
        }
    }
}