Initial commit
This commit is contained in:
commit
6b4ba28078
|
@ -0,0 +1,19 @@
|
||||||
|
# This file is for unifying the coding style for different editors and IDEs
|
||||||
|
# editorconfig.org
|
||||||
|
#
|
||||||
|
# Subline Text: https://github.com/editorconfig/editorconfig-sublime
|
||||||
|
# Emacs: https://github.com/editorconfig/editorconfig-emacs
|
||||||
|
# Jetbrains (IntelliJ etc): https://github.com/editorconfig/editorconfig-jetbrains
|
||||||
|
# Eclipse: ??
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
end_of_line = lf
|
||||||
|
charset = utf-8
|
||||||
|
trim_trailing_whitespace = true
|
||||||
|
insert_final_newline = true
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
|
@ -0,0 +1,3 @@
|
||||||
|
.idea/
|
||||||
|
target/
|
||||||
|
*.iml
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!groovy
|
||||||
|
|
||||||
|
node {
|
||||||
|
stage 'Build and Test'
|
||||||
|
checkout scm
|
||||||
|
sh 'mvn clean package'
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2016 CloudBees Inc and a number of other of contributors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
|
@ -0,0 +1,3 @@
|
||||||
|
= Plugin Site API
|
||||||
|
|
||||||
|
Backend API for Jenkins Plugin Site
|
|
@ -0,0 +1,137 @@
|
||||||
|
<project
|
||||||
|
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>io.jenkins</groupId>
|
||||||
|
<artifactId>plugins</artifactId>
|
||||||
|
<packaging>war</packaging>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<jersey.version>2.23.1</jersey.version>
|
||||||
|
<slf4j.version>1.7.21</slf4j.version>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencyManagement>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.glassfish.jersey</groupId>
|
||||||
|
<artifactId>jersey-bom</artifactId>
|
||||||
|
<version>${jersey.version}</version>
|
||||||
|
<type>pom</type>
|
||||||
|
<scope>import</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</dependencyManagement>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-io</groupId>
|
||||||
|
<artifactId>commons-io</artifactId>
|
||||||
|
<version>2.5</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.httpcomponents</groupId>
|
||||||
|
<artifactId>httpclient</artifactId>
|
||||||
|
<version>4.5.2</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.json</groupId>
|
||||||
|
<artifactId>json</artifactId>
|
||||||
|
<version>20160212</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.elasticsearch</groupId>
|
||||||
|
<artifactId>elasticsearch</artifactId>
|
||||||
|
<version>1.7.5</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.glassfish.jersey.containers</groupId>
|
||||||
|
<artifactId>jersey-container-servlet</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.glassfish.jersey.media</groupId>
|
||||||
|
<artifactId>jersey-media-json-jackson</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.quartz-scheduler</groupId>
|
||||||
|
<artifactId>quartz</artifactId>
|
||||||
|
<version>2.2.3</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
<version>${slf4j.version}</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
<version>1.1.7</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>javax.servlet</groupId>
|
||||||
|
<artifactId>javax.servlet-api</artifactId>
|
||||||
|
<version>3.1.0</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.12</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-clean-plugin</artifactId>
|
||||||
|
<version>3.0.0</version>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.5.1</version>
|
||||||
|
<configuration>
|
||||||
|
<source>1.8</source>
|
||||||
|
<target>1.8</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-resources-plugin</artifactId>
|
||||||
|
<version>3.0.1</version>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-surefire-plugin</artifactId>
|
||||||
|
<version>2.19.1</version>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-war-plugin</artifactId>
|
||||||
|
<version>2.6</version>
|
||||||
|
<configuration>
|
||||||
|
<failOnMissingWebXml>false</failOnMissingWebXml>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.eclipse.jetty</groupId>
|
||||||
|
<artifactId>jetty-maven-plugin</artifactId>
|
||||||
|
<version>9.3.10.v20160621</version>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
<scm>
|
||||||
|
<developerConnection>scm:git:ssh://git@github.com/jenkins-infra/plugin-site.git</developerConnection>
|
||||||
|
<connection>scm:git:ssh://git@github.com/jenkins-infra/plugin-site.git</connection>
|
||||||
|
</scm>
|
||||||
|
|
||||||
|
</project>
|
|
@ -0,0 +1,35 @@
|
||||||
|
package io.jenkins.plugins;
|
||||||
|
|
||||||
|
import io.jenkins.plugins.db.support.ElasticsearchClientFactory;
|
||||||
|
import io.jenkins.plugins.db.support.EmbeddedElasticsearchServer;
|
||||||
|
import io.jenkins.plugins.schedule.JobScheduler;
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.glassfish.hk2.api.ServiceLocator;
|
||||||
|
import org.glassfish.hk2.utilities.ServiceLocatorUtilities;
|
||||||
|
import org.glassfish.hk2.utilities.binding.AbstractBinder;
|
||||||
|
import org.glassfish.jersey.server.ResourceConfig;
|
||||||
|
|
||||||
|
import javax.inject.Inject;
|
||||||
|
import javax.inject.Singleton;
|
||||||
|
import javax.ws.rs.ApplicationPath;
|
||||||
|
|
||||||
|
@ApplicationPath("/")
|
||||||
|
public class RestApp extends ResourceConfig {
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
public RestApp(ServiceLocator locator) {
|
||||||
|
ServiceLocatorUtilities.enableImmediateScope(locator);
|
||||||
|
|
||||||
|
register(new AbstractBinder() {
|
||||||
|
@Override
|
||||||
|
protected void configure() {
|
||||||
|
bind(EmbeddedElasticsearchServer.class).to(EmbeddedElasticsearchServer.class).in(Singleton.class);
|
||||||
|
bindFactory(ElasticsearchClientFactory.class).to(Client.class).in(Singleton.class);
|
||||||
|
bind(JobScheduler.class).to(JobScheduler.class).in(Singleton.class);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
packages("io.jenkins.plugins");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,25 @@
|
||||||
|
package io.jenkins.plugins.db.support;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.glassfish.hk2.api.Factory;
|
||||||
|
import org.glassfish.hk2.api.Immediate;
|
||||||
|
|
||||||
|
import javax.inject.Inject;
|
||||||
|
import javax.inject.Singleton;
|
||||||
|
|
||||||
|
@Immediate
|
||||||
|
public class ElasticsearchClientFactory implements Factory<Client> {
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
private EmbeddedElasticsearchServer es;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Client provide() {
|
||||||
|
return es.getClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void dispose(Client client) {
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,78 @@
|
||||||
|
package io.jenkins.plugins.db.support;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.node.Node;
|
||||||
|
import org.elasticsearch.node.NodeBuilder;
|
||||||
|
import org.glassfish.hk2.api.Immediate;
|
||||||
|
import org.jvnet.hk2.annotations.Service;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.annotation.PostConstruct;
|
||||||
|
import javax.annotation.PreDestroy;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Copied & modified from:
|
||||||
|
* https://orrsella.com/2014/10/28/embedded-elasticsearch-server-for-scala-integration-tests/
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
@Immediate
|
||||||
|
public class EmbeddedElasticsearchServer {
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(EmbeddedElasticsearchServer.class);
|
||||||
|
|
||||||
|
private File dataDir;
|
||||||
|
private Node node;
|
||||||
|
|
||||||
|
public Client getClient() {
|
||||||
|
return node.client();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void postConstruct() {
|
||||||
|
logger.info("Initialize elasticsearch");
|
||||||
|
try {
|
||||||
|
this.dataDir = Files.createTempDirectory("elasticsearch_data").toFile();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Problem creating temp data directory", e);
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
final Settings settings = ImmutableSettings.settingsBuilder()
|
||||||
|
.put("path.data", dataDir)
|
||||||
|
.put("http.enabled", "false")
|
||||||
|
.build();
|
||||||
|
node = NodeBuilder.nodeBuilder().local(true).settings(settings).build();
|
||||||
|
node.start();
|
||||||
|
createIndex();
|
||||||
|
logger.info("Initializing elasticsearch done");
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void preDestroy() {
|
||||||
|
node.close();
|
||||||
|
FileUtils.deleteQuietly(this.dataDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void createIndex() {
|
||||||
|
try {
|
||||||
|
final ClassLoader cl = getClass().getClassLoader();
|
||||||
|
final File file = new File(cl.getResource("elasticsearch/plugins.json").getFile());
|
||||||
|
final String body = FileUtils.readFileToString(file, "utf-8");
|
||||||
|
final Client client = getClient();
|
||||||
|
client.admin().indices().prepareCreate("plugins")
|
||||||
|
.addMapping("plugins", body)
|
||||||
|
.get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("Problem creating index", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,28 @@
|
||||||
|
package io.jenkins.plugins.endpoints;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
|
||||||
|
import javax.ws.rs.GET;
|
||||||
|
import javax.ws.rs.Path;
|
||||||
|
import javax.ws.rs.Produces;
|
||||||
|
import javax.ws.rs.WebApplicationException;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
@Path("/categories")
|
||||||
|
@Produces(MediaType.APPLICATION_JSON)
|
||||||
|
public class Categories {
|
||||||
|
|
||||||
|
@GET
|
||||||
|
public String getCategories() {
|
||||||
|
try {
|
||||||
|
final ClassLoader cl = getClass().getClassLoader();
|
||||||
|
final File file = new File(cl.getResource("categories.json").getFile());
|
||||||
|
return FileUtils.readFileToString(file, "utf-8");
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,38 @@
|
||||||
|
package io.jenkins.plugins.endpoints;
|
||||||
|
|
||||||
|
import org.elasticsearch.action.get.GetResponse;
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.inject.Inject;
|
||||||
|
import javax.ws.rs.*;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
|
||||||
|
@Path("/plugin/{name}")
|
||||||
|
@Produces(MediaType.APPLICATION_JSON)
|
||||||
|
public class Plugin {
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(Plugin.class);
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
private Client esClient;
|
||||||
|
|
||||||
|
@GET
|
||||||
|
public String getPlugin(@PathParam("name") String name) {
|
||||||
|
try {
|
||||||
|
final GetResponse getResponse = esClient.prepareGet("plugins", "plugins", name).execute().get();
|
||||||
|
if (getResponse.isExists()) {
|
||||||
|
return getResponse.getSourceAsString();
|
||||||
|
} else {
|
||||||
|
throw new WebApplicationException(Response.Status.NOT_FOUND);
|
||||||
|
}
|
||||||
|
} catch (InterruptedException | ExecutionException e) {
|
||||||
|
logger.error("Problem getting plugin " + name, e);
|
||||||
|
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,68 @@
|
||||||
|
package io.jenkins.plugins.endpoints;
|
||||||
|
|
||||||
|
import io.jenkins.plugins.schedule.JobScheduler;
|
||||||
|
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
|
import org.glassfish.hk2.api.Immediate;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.inject.Inject;
|
||||||
|
import javax.ws.rs.*;
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
import javax.ws.rs.core.Response;
|
||||||
|
|
||||||
|
@Path("/plugins")
|
||||||
|
@Produces(MediaType.APPLICATION_JSON)
|
||||||
|
@Immediate
|
||||||
|
public class Plugins {
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(Plugins.class);
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
private Client esClient;
|
||||||
|
|
||||||
|
// Hackity hack hack. This is only here because I can't figure out how to successfully
|
||||||
|
// get the JobScheduler in RestApp from the ServiceLocator. Whenever I try I get NPE
|
||||||
|
// so this injection at least makes @PostConstruct fire
|
||||||
|
@Inject
|
||||||
|
private JobScheduler jobScheduler;
|
||||||
|
|
||||||
|
@GET
|
||||||
|
public String search(
|
||||||
|
@QueryParam("q") String query,
|
||||||
|
@DefaultValue("name") @QueryParam("sort") String sort,
|
||||||
|
@DefaultValue("") @QueryParam("labels") String labels,
|
||||||
|
@DefaultValue("") @QueryParam("authors")String authors,
|
||||||
|
@DefaultValue("") @QueryParam("core")String core,
|
||||||
|
@DefaultValue("50") @QueryParam("size") int size,
|
||||||
|
@DefaultValue("1") @QueryParam("page") int page) {
|
||||||
|
try {
|
||||||
|
final SearchRequestBuilder requestBuilder = esClient.prepareSearch("plugins")
|
||||||
|
.addFields("name", "url", "title", "wiki", "excerpt", "labels")
|
||||||
|
.addHighlightedField("excerpt")
|
||||||
|
.setHighlighterFragmentSize(500)
|
||||||
|
.setHighlighterNumOfFragments(1)
|
||||||
|
.setHighlighterPreTags("<mark>")
|
||||||
|
.setHighlighterPostTags("</mark>")
|
||||||
|
.setFrom((page -1) * size)
|
||||||
|
.setSize(size);
|
||||||
|
final BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||||
|
.should(QueryBuilders.matchQuery("title", query))
|
||||||
|
.should(QueryBuilders.matchQuery("name", query))
|
||||||
|
.must(QueryBuilders.matchQuery("excerpt", query));
|
||||||
|
if (!labels.isEmpty()) {
|
||||||
|
queryBuilder.should(QueryBuilders.termsQuery("labels", labels));
|
||||||
|
}
|
||||||
|
final SearchResponse response = requestBuilder.setQuery(queryBuilder).execute().get();
|
||||||
|
return response.toString();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("Problem executing ES query", e);
|
||||||
|
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,78 @@
|
||||||
|
package io.jenkins.plugins.schedule;
|
||||||
|
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.glassfish.hk2.api.Immediate;
|
||||||
|
import org.jvnet.hk2.annotations.Service;
|
||||||
|
import org.quartz.*;
|
||||||
|
import org.quartz.impl.StdSchedulerFactory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import javax.annotation.PostConstruct;
|
||||||
|
import javax.annotation.PreDestroy;
|
||||||
|
import javax.inject.Inject;
|
||||||
|
|
||||||
|
@Service
|
||||||
|
@Immediate
|
||||||
|
public class JobScheduler {
|
||||||
|
|
||||||
|
private static final JobKey JOB_KEY = JobKey.jobKey("populateElasticsearchJob");
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(JobScheduler.class);
|
||||||
|
|
||||||
|
private Scheduler scheduler;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
private Client esClient;
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void postConstruct() {
|
||||||
|
try {
|
||||||
|
this.scheduler = StdSchedulerFactory.getDefaultScheduler();
|
||||||
|
this.scheduler.start();
|
||||||
|
} catch (SchedulerException e) {
|
||||||
|
logger.error("Problem initializing Quartz", e);
|
||||||
|
}
|
||||||
|
schedule();
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
public void preDestroy() {
|
||||||
|
try {
|
||||||
|
if (!this.scheduler.isShutdown()) {
|
||||||
|
this.scheduler.shutdown();
|
||||||
|
}
|
||||||
|
} catch (SchedulerException e) {
|
||||||
|
logger.error("Problem shutting down Quartz", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void schedule() {
|
||||||
|
final JobDataMap dataMap = new JobDataMap();
|
||||||
|
dataMap.put(PopulateElasticsearchJob.ES_CLIENT_KEY, esClient);
|
||||||
|
final JobDetail job = JobBuilder
|
||||||
|
.newJob(PopulateElasticsearchJob.class)
|
||||||
|
.withIdentity(JOB_KEY)
|
||||||
|
.usingJobData(dataMap)
|
||||||
|
.build();
|
||||||
|
final Trigger trigger = TriggerBuilder
|
||||||
|
.newTrigger()
|
||||||
|
.withIdentity("populateElasticserchTrigger")
|
||||||
|
// Fire now and then every 12 hours
|
||||||
|
.withSchedule(SimpleScheduleBuilder.simpleSchedule()
|
||||||
|
.withIntervalInHours(12)
|
||||||
|
.repeatForever()
|
||||||
|
)
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
if (!scheduler.checkExists(JOB_KEY)) {
|
||||||
|
scheduler.scheduleJob(job, trigger);
|
||||||
|
} else {
|
||||||
|
logger.info("Already scheduled");
|
||||||
|
}
|
||||||
|
} catch (SchedulerException e) {
|
||||||
|
logger.error("Problem scheduling " + JOB_KEY + " + job");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,160 @@
|
||||||
|
package io.jenkins.plugins.schedule;
|
||||||
|
|
||||||
|
import org.apache.http.HttpEntity;
|
||||||
|
import org.apache.http.StatusLine;
|
||||||
|
import org.apache.http.client.ClientProtocolException;
|
||||||
|
import org.apache.http.client.ResponseHandler;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
import org.elasticsearch.action.bulk.BulkProcessor;
|
||||||
|
import org.elasticsearch.action.bulk.BulkRequest;
|
||||||
|
import org.elasticsearch.action.bulk.BulkResponse;
|
||||||
|
import org.elasticsearch.client.Client;
|
||||||
|
import org.json.JSONArray;
|
||||||
|
import org.json.JSONObject;
|
||||||
|
import org.quartz.Job;
|
||||||
|
import org.quartz.JobExecutionContext;
|
||||||
|
import org.quartz.JobExecutionException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
public class PopulateElasticsearchJob implements Job {
|
||||||
|
|
||||||
|
public static String ES_CLIENT_KEY = "esClient";
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(PopulateElasticsearchJob.class);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void execute(JobExecutionContext context) throws JobExecutionException {
|
||||||
|
final Client esClient = (Client)context.getJobDetail().getJobDataMap().get(ES_CLIENT_KEY);
|
||||||
|
if (Objects.isNull(esClient)) {
|
||||||
|
throw new JobExecutionException(ES_CLIENT_KEY + " is null");
|
||||||
|
}
|
||||||
|
logger.info("Starting PopulateElasticsearchJob");
|
||||||
|
final int processors = Runtime.getRuntime().availableProcessors();
|
||||||
|
final ExecutorService executorService = Executors.newFixedThreadPool(processors * 10);
|
||||||
|
final CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||||
|
try {
|
||||||
|
final ResponseHandler<JSONObject> updateCenterHandler = (httpResponse) -> {
|
||||||
|
final StatusLine status = httpResponse.getStatusLine();
|
||||||
|
if (status.getStatusCode() == 200) {
|
||||||
|
final HttpEntity entity = httpResponse.getEntity();
|
||||||
|
final String content = EntityUtils.toString(entity);
|
||||||
|
try {
|
||||||
|
return new JSONObject(String.join("", content.split("\n")[1])).getJSONObject("plugins");
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ClientProtocolException("Update center returned invalid JSON");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new ClientProtocolException("Unexpected response from update center - " + status.toString());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
final JSONObject plugins = httpClient.execute(new HttpGet("https://updates.jenkins-ci.org/current/update-center.json"), updateCenterHandler);
|
||||||
|
final List<Callable<Void>> tasks = new ArrayList<>();
|
||||||
|
for (String key : plugins.keySet()) {
|
||||||
|
final JSONObject plugin = plugins.getJSONObject(key);
|
||||||
|
final Callable<Void> task = () -> {
|
||||||
|
final ResponseHandler<Void> statsHandler = (httpResponse) -> {
|
||||||
|
final StatusLine status = httpResponse.getStatusLine();
|
||||||
|
if (status.getStatusCode() == 200) {
|
||||||
|
final HttpEntity entity = httpResponse.getEntity();
|
||||||
|
final String content = EntityUtils.toString(entity);
|
||||||
|
try {
|
||||||
|
final JSONObject stats = new JSONObject(content);
|
||||||
|
final JSONObject installations = stats.getJSONObject("installations");
|
||||||
|
final JSONObject installationsPercentage = stats.getJSONObject("installationsPercentage");
|
||||||
|
final JSONObject installationsPerVersion = stats.getJSONObject("installationsPerVersion");
|
||||||
|
final JSONObject installationsPercentagePerVersion = stats.getJSONObject("installationsPercentagePerVersion");
|
||||||
|
plugin.put("installations", new JSONArray(installations.keySet().stream().map((timestamp) -> {
|
||||||
|
final JSONObject installation = new JSONObject();
|
||||||
|
installation.put("timestamp", Long.valueOf(timestamp));
|
||||||
|
installation.put("total", installations.getInt(timestamp));
|
||||||
|
return installation;
|
||||||
|
}).collect(Collectors.toSet())));
|
||||||
|
plugin.put("installationsPercentage", new JSONArray(installationsPercentage.keySet().stream().map((timestamp) -> {
|
||||||
|
final JSONObject installation = new JSONObject();
|
||||||
|
installation.put("timestamp", Long.valueOf(timestamp));
|
||||||
|
installation.put("percentage", installationsPercentage.getDouble(timestamp));
|
||||||
|
return installation;
|
||||||
|
}).collect(Collectors.toSet())));
|
||||||
|
plugin.put("installationsPerVersion", new JSONArray(installationsPerVersion.keySet().stream().map((version) -> {
|
||||||
|
final JSONObject installation = new JSONObject();
|
||||||
|
installation.put("version", version);
|
||||||
|
installation.put("total", installationsPerVersion.getInt(version));
|
||||||
|
return installation;
|
||||||
|
}).collect(Collectors.toSet())));
|
||||||
|
plugin.put("installationsPercentagePerVersion", new JSONArray(installationsPercentagePerVersion.keySet().stream().map((version) -> {
|
||||||
|
final JSONObject installation = new JSONObject();
|
||||||
|
installation.put("version", version);
|
||||||
|
installation.put("total", installationsPercentagePerVersion.getDouble(version));
|
||||||
|
return installation;
|
||||||
|
}).collect(Collectors.toSet())));
|
||||||
|
return null;
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ClientProtocolException("Problem processing stats", e);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new ClientProtocolException("Unexpected response for stats " + status.toString());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
final String url = "http://stats.jenkins-ci.org/plugin-installation-trend/" + plugin.get("name") + ".stats.json";
|
||||||
|
try {
|
||||||
|
httpClient.execute(new HttpGet(url), statsHandler);
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("Problem getting status for plugin " + key);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
tasks.add(task);
|
||||||
|
}
|
||||||
|
executorService.invokeAll(tasks).stream().map(task -> {
|
||||||
|
try {
|
||||||
|
return task.get();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IllegalStateException(e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
final BulkProcessor bulk = BulkProcessor.builder(esClient, new BulkProcessor.Listener() {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void beforeBulk(long executionId, BulkRequest request) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||||
|
logger.info("Indexed " + response.getItems().length + " plugins in " + response.getTook().minutes() + " min " + response.getTook().seconds() + " sec");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
|
||||||
|
}
|
||||||
|
}).build();
|
||||||
|
plugins.keySet().forEach((key) -> {
|
||||||
|
final JSONObject plugin = plugins.getJSONObject(key);
|
||||||
|
bulk.add(esClient.prepareIndex("plugins", "plugins").setId(plugin.getString("name")).setSource(plugin.toString()).request());
|
||||||
|
});
|
||||||
|
bulk.awaitClose(2, TimeUnit.MINUTES);
|
||||||
|
logger.info("Finished PopulateElasticsearchJob");
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("Problem getting plugin information", e);
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
httpClient.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Problem closing httpClient", e);
|
||||||
|
}
|
||||||
|
if (!executorService.isShutdown()) {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,186 @@
|
||||||
|
{
|
||||||
|
"categories":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "languagesPlatforms",
|
||||||
|
"title": "Platforms",
|
||||||
|
"description": "Jenkins plugins that are designed to give added support for building, testing or deploying to specific languages or platforms.",
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "ios",
|
||||||
|
"title": "iOS development"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "dotnet",
|
||||||
|
"title": "Azure and .NET"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "android",
|
||||||
|
"title": "Android development"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "ruby",
|
||||||
|
"title": "Ruby development"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "scala",
|
||||||
|
"title": "Scala plugins"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "userInterface",
|
||||||
|
"title": "User interface",
|
||||||
|
"description": "Add-ons designed to make your iteraction with Jenkins better.",
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "ui",
|
||||||
|
"title": "User Interface"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "listview-column",
|
||||||
|
"title": "List view column plugins"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "adminTools",
|
||||||
|
"title": "Administration",
|
||||||
|
"description": "Add functionality to help streamline your administration of Jenkins.",
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "slaves",
|
||||||
|
"title": "Agent launchers and controllers"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "page-decorator",
|
||||||
|
"title": "Page decorators"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "user",
|
||||||
|
"title": "Authentication and user management"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "cluster",
|
||||||
|
"title": "Cluster management and distributed build"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "cli",
|
||||||
|
"title": "CLI extensions"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "buildManagement",
|
||||||
|
"title": "Build management",
|
||||||
|
"description": "Add new means of trigger jobs and new ways to ensure environment consistency for you automated processes.",
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "trigger",
|
||||||
|
"title": "Build triggers"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "buildwrapper",
|
||||||
|
"title": "Build wrappers"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "notifier",
|
||||||
|
"title": "Build notifiers"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "deployment",
|
||||||
|
"title": "Deployment plugins"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "parameter",
|
||||||
|
"title": "Build parameters"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "post-build",
|
||||||
|
"title": "Clean-up actions"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "builder",
|
||||||
|
"title": "Build tools"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "report",
|
||||||
|
"title": "Build reports"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "upload",
|
||||||
|
"title": "Artifact uploaders"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "scm",
|
||||||
|
"title": "Source code management",
|
||||||
|
"description": "Connect to your Jenkins to your SCM.",
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "scm",
|
||||||
|
"title": "Source Code Management"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "scm-related",
|
||||||
|
"title": "Source Code Management related"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "junk",
|
||||||
|
"hidden": true,
|
||||||
|
"labels":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": "library"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "misc"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "must-be-labeled"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "external"
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "maven"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
|
@ -0,0 +1,141 @@
|
||||||
|
{
|
||||||
|
"plugins" : {
|
||||||
|
"properties" : {
|
||||||
|
"buildDate" : {
|
||||||
|
"type" : "date",
|
||||||
|
"format" : "MMM dd, YYYY"
|
||||||
|
},
|
||||||
|
"dependencies" : {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"name" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"optional" : {
|
||||||
|
"type" : "boolean"
|
||||||
|
},
|
||||||
|
"version" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"developers" : {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"developerId" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"email" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"name" : {
|
||||||
|
"type" : "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"exerpt" : {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"gav" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"installations" : {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"timestamp" : {
|
||||||
|
"type" : "long"
|
||||||
|
},
|
||||||
|
"total" : {
|
||||||
|
"type" : "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"installationsPercentage": {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"timestamp" : {
|
||||||
|
"type" : "long"
|
||||||
|
},
|
||||||
|
"percentage" : {
|
||||||
|
"type" : "double"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"installationsPerVersion" : {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"version" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"total" : {
|
||||||
|
"type" : "integer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"installationsPercentagePerVersion" : {
|
||||||
|
"type" : "nested",
|
||||||
|
"properties" : {
|
||||||
|
"version" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"percentage" : {
|
||||||
|
"type" : "double"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"labels" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"name" : {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"previousTimestamp" : {
|
||||||
|
"type" : "date",
|
||||||
|
"format" : "date_time"
|
||||||
|
},
|
||||||
|
"previousVersion" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"releaseTimestamp" : {
|
||||||
|
"type" : "date",
|
||||||
|
"format" : "date_time"
|
||||||
|
},
|
||||||
|
"requiredCore" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"scm" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"sha1" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"title" : {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"url" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"version" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
},
|
||||||
|
"wiki" : {
|
||||||
|
"type" : "string",
|
||||||
|
"index" : "not_analyzed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,16 @@
|
||||||
|
<configuration>
|
||||||
|
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="debug">
|
||||||
|
<appender-ref ref="STDOUT" />
|
||||||
|
</root>
|
||||||
|
|
||||||
|
<logger name="org.elasticsearch" level="error"/>
|
||||||
|
<logger name="org.quartz" level="error"/>
|
||||||
|
|
||||||
|
</configuration>
|
Loading…
Reference in New Issue