Initial commit
This commit is contained in:
Michael McCaskill
commit
6b4ba28078
|
|
@ -0,0 +1,19 @@
|
|||
# This file is for unifying the coding style for different editors and IDEs
|
||||
# editorconfig.org
|
||||
#
|
||||
# Subline Text: https://github.com/editorconfig/editorconfig-sublime
|
||||
# Emacs: https://github.com/editorconfig/editorconfig-emacs
|
||||
# Jetbrains (IntelliJ etc): https://github.com/editorconfig/editorconfig-jetbrains
|
||||
# Eclipse: ??
|
||||
#
|
||||
|
||||
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
.idea/
|
||||
target/
|
||||
*.iml
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!groovy
|
||||
|
||||
node {
|
||||
stage 'Build and Test'
|
||||
checkout scm
|
||||
sh 'mvn clean package'
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2016 CloudBees Inc and a number of other of contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
= Plugin Site API
|
||||
|
||||
Backend API for Jenkins Plugin Site
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
<project
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>io.jenkins</groupId>
|
||||
<artifactId>plugins</artifactId>
|
||||
<packaging>war</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<jersey.version>2.23.1</jersey.version>
|
||||
<slf4j.version>1.7.21</slf4j.version>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey</groupId>
|
||||
<artifactId>jersey-bom</artifactId>
|
||||
<version>${jersey.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<version>2.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20160212</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch</artifactId>
|
||||
<version>1.7.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.containers</groupId>
|
||||
<artifactId>jersey-container-servlet</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.media</groupId>
|
||||
<artifactId>jersey-media-json-jackson</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.quartz-scheduler</groupId>
|
||||
<artifactId>quartz</artifactId>
|
||||
<version>2.2.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
<version>1.1.7</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>javax.servlet</groupId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<version>3.1.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<version>3.0.0</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.5.1</version>
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<version>3.0.1</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.19.1</version>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-war-plugin</artifactId>
|
||||
<version>2.6</version>
|
||||
<configuration>
|
||||
<failOnMissingWebXml>false</failOnMissingWebXml>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-maven-plugin</artifactId>
|
||||
<version>9.3.10.v20160621</version>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<scm>
|
||||
<developerConnection>scm:git:ssh://git@github.com/jenkins-infra/plugin-site.git</developerConnection>
|
||||
<connection>scm:git:ssh://git@github.com/jenkins-infra/plugin-site.git</connection>
|
||||
</scm>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package io.jenkins.plugins;
|
||||
|
||||
import io.jenkins.plugins.db.support.ElasticsearchClientFactory;
|
||||
import io.jenkins.plugins.db.support.EmbeddedElasticsearchServer;
|
||||
import io.jenkins.plugins.schedule.JobScheduler;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.glassfish.hk2.api.ServiceLocator;
|
||||
import org.glassfish.hk2.utilities.ServiceLocatorUtilities;
|
||||
import org.glassfish.hk2.utilities.binding.AbstractBinder;
|
||||
import org.glassfish.jersey.server.ResourceConfig;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
import javax.ws.rs.ApplicationPath;
|
||||
|
||||
@ApplicationPath("/")
|
||||
public class RestApp extends ResourceConfig {
|
||||
|
||||
@Inject
|
||||
public RestApp(ServiceLocator locator) {
|
||||
ServiceLocatorUtilities.enableImmediateScope(locator);
|
||||
|
||||
register(new AbstractBinder() {
|
||||
@Override
|
||||
protected void configure() {
|
||||
bind(EmbeddedElasticsearchServer.class).to(EmbeddedElasticsearchServer.class).in(Singleton.class);
|
||||
bindFactory(ElasticsearchClientFactory.class).to(Client.class).in(Singleton.class);
|
||||
bind(JobScheduler.class).to(JobScheduler.class).in(Singleton.class);
|
||||
}
|
||||
});
|
||||
|
||||
packages("io.jenkins.plugins");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package io.jenkins.plugins.db.support;
|
||||
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.glassfish.hk2.api.Factory;
|
||||
import org.glassfish.hk2.api.Immediate;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
|
||||
@Immediate
|
||||
public class ElasticsearchClientFactory implements Factory<Client> {
|
||||
|
||||
@Inject
|
||||
private EmbeddedElasticsearchServer es;
|
||||
|
||||
@Override
|
||||
public Client provide() {
|
||||
return es.getClient();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dispose(Client client) {
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package io.jenkins.plugins.db.support;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.node.NodeBuilder;
|
||||
import org.glassfish.hk2.api.Immediate;
|
||||
import org.jvnet.hk2.annotations.Service;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
/**
|
||||
* Copied & modified from:
|
||||
* https://orrsella.com/2014/10/28/embedded-elasticsearch-server-for-scala-integration-tests/
|
||||
*/
|
||||
@Service
|
||||
@Immediate
|
||||
public class EmbeddedElasticsearchServer {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(EmbeddedElasticsearchServer.class);
|
||||
|
||||
private File dataDir;
|
||||
private Node node;
|
||||
|
||||
public Client getClient() {
|
||||
return node.client();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void postConstruct() {
|
||||
logger.info("Initialize elasticsearch");
|
||||
try {
|
||||
this.dataDir = Files.createTempDirectory("elasticsearch_data").toFile();
|
||||
} catch (IOException e) {
|
||||
logger.error("Problem creating temp data directory", e);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
final Settings settings = ImmutableSettings.settingsBuilder()
|
||||
.put("path.data", dataDir)
|
||||
.put("http.enabled", "false")
|
||||
.build();
|
||||
node = NodeBuilder.nodeBuilder().local(true).settings(settings).build();
|
||||
node.start();
|
||||
createIndex();
|
||||
logger.info("Initializing elasticsearch done");
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void preDestroy() {
|
||||
node.close();
|
||||
FileUtils.deleteQuietly(this.dataDir);
|
||||
}
|
||||
|
||||
public void createIndex() {
|
||||
try {
|
||||
final ClassLoader cl = getClass().getClassLoader();
|
||||
final File file = new File(cl.getResource("elasticsearch/plugins.json").getFile());
|
||||
final String body = FileUtils.readFileToString(file, "utf-8");
|
||||
final Client client = getClient();
|
||||
client.admin().indices().prepareCreate("plugins")
|
||||
.addMapping("plugins", body)
|
||||
.get();
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem creating index", e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package io.jenkins.plugins.endpoints;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.WebApplicationException;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
import java.io.File;
|
||||
|
||||
@Path("/categories")
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
public class Categories {
|
||||
|
||||
@GET
|
||||
public String getCategories() {
|
||||
try {
|
||||
final ClassLoader cl = getClass().getClassLoader();
|
||||
final File file = new File(cl.getResource("categories.json").getFile());
|
||||
return FileUtils.readFileToString(file, "utf-8");
|
||||
} catch (Exception e) {
|
||||
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package io.jenkins.plugins.endpoints;
|
||||
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.ws.rs.*;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
@Path("/plugin/{name}")
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
public class Plugin {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(Plugin.class);
|
||||
|
||||
@Inject
|
||||
private Client esClient;
|
||||
|
||||
@GET
|
||||
public String getPlugin(@PathParam("name") String name) {
|
||||
try {
|
||||
final GetResponse getResponse = esClient.prepareGet("plugins", "plugins", name).execute().get();
|
||||
if (getResponse.isExists()) {
|
||||
return getResponse.getSourceAsString();
|
||||
} else {
|
||||
throw new WebApplicationException(Response.Status.NOT_FOUND);
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
logger.error("Problem getting plugin " + name, e);
|
||||
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package io.jenkins.plugins.endpoints;
|
||||
|
||||
import io.jenkins.plugins.schedule.JobScheduler;
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.glassfish.hk2.api.Immediate;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.ws.rs.*;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
|
||||
@Path("/plugins")
|
||||
@Produces(MediaType.APPLICATION_JSON)
|
||||
@Immediate
|
||||
public class Plugins {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(Plugins.class);
|
||||
|
||||
@Inject
|
||||
private Client esClient;
|
||||
|
||||
// Hackity hack hack. This is only here because I can't figure out how to successfully
|
||||
// get the JobScheduler in RestApp from the ServiceLocator. Whenever I try I get NPE
|
||||
// so this injection at least makes @PostConstruct fire
|
||||
@Inject
|
||||
private JobScheduler jobScheduler;
|
||||
|
||||
@GET
|
||||
public String search(
|
||||
@QueryParam("q") String query,
|
||||
@DefaultValue("name") @QueryParam("sort") String sort,
|
||||
@DefaultValue("") @QueryParam("labels") String labels,
|
||||
@DefaultValue("") @QueryParam("authors")String authors,
|
||||
@DefaultValue("") @QueryParam("core")String core,
|
||||
@DefaultValue("50") @QueryParam("size") int size,
|
||||
@DefaultValue("1") @QueryParam("page") int page) {
|
||||
try {
|
||||
final SearchRequestBuilder requestBuilder = esClient.prepareSearch("plugins")
|
||||
.addFields("name", "url", "title", "wiki", "excerpt", "labels")
|
||||
.addHighlightedField("excerpt")
|
||||
.setHighlighterFragmentSize(500)
|
||||
.setHighlighterNumOfFragments(1)
|
||||
.setHighlighterPreTags("<mark>")
|
||||
.setHighlighterPostTags("</mark>")
|
||||
.setFrom((page -1) * size)
|
||||
.setSize(size);
|
||||
final BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.should(QueryBuilders.matchQuery("title", query))
|
||||
.should(QueryBuilders.matchQuery("name", query))
|
||||
.must(QueryBuilders.matchQuery("excerpt", query));
|
||||
if (!labels.isEmpty()) {
|
||||
queryBuilder.should(QueryBuilders.termsQuery("labels", labels));
|
||||
}
|
||||
final SearchResponse response = requestBuilder.setQuery(queryBuilder).execute().get();
|
||||
return response.toString();
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem executing ES query", e);
|
||||
throw new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package io.jenkins.plugins.schedule;
|
||||
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.glassfish.hk2.api.Immediate;
|
||||
import org.jvnet.hk2.annotations.Service;
|
||||
import org.quartz.*;
|
||||
import org.quartz.impl.StdSchedulerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.annotation.PreDestroy;
|
||||
import javax.inject.Inject;
|
||||
|
||||
@Service
|
||||
@Immediate
|
||||
public class JobScheduler {
|
||||
|
||||
private static final JobKey JOB_KEY = JobKey.jobKey("populateElasticsearchJob");
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(JobScheduler.class);
|
||||
|
||||
private Scheduler scheduler;
|
||||
|
||||
@Inject
|
||||
private Client esClient;
|
||||
|
||||
@PostConstruct
|
||||
public void postConstruct() {
|
||||
try {
|
||||
this.scheduler = StdSchedulerFactory.getDefaultScheduler();
|
||||
this.scheduler.start();
|
||||
} catch (SchedulerException e) {
|
||||
logger.error("Problem initializing Quartz", e);
|
||||
}
|
||||
schedule();
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void preDestroy() {
|
||||
try {
|
||||
if (!this.scheduler.isShutdown()) {
|
||||
this.scheduler.shutdown();
|
||||
}
|
||||
} catch (SchedulerException e) {
|
||||
logger.error("Problem shutting down Quartz", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void schedule() {
|
||||
final JobDataMap dataMap = new JobDataMap();
|
||||
dataMap.put(PopulateElasticsearchJob.ES_CLIENT_KEY, esClient);
|
||||
final JobDetail job = JobBuilder
|
||||
.newJob(PopulateElasticsearchJob.class)
|
||||
.withIdentity(JOB_KEY)
|
||||
.usingJobData(dataMap)
|
||||
.build();
|
||||
final Trigger trigger = TriggerBuilder
|
||||
.newTrigger()
|
||||
.withIdentity("populateElasticserchTrigger")
|
||||
// Fire now and then every 12 hours
|
||||
.withSchedule(SimpleScheduleBuilder.simpleSchedule()
|
||||
.withIntervalInHours(12)
|
||||
.repeatForever()
|
||||
)
|
||||
.build();
|
||||
try {
|
||||
if (!scheduler.checkExists(JOB_KEY)) {
|
||||
scheduler.scheduleJob(job, trigger);
|
||||
} else {
|
||||
logger.info("Already scheduled");
|
||||
}
|
||||
} catch (SchedulerException e) {
|
||||
logger.error("Problem scheduling " + JOB_KEY + " + job");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
package io.jenkins.plugins.schedule;
|
||||
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.StatusLine;
|
||||
import org.apache.http.client.ClientProtocolException;
|
||||
import org.apache.http.client.ResponseHandler;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.elasticsearch.action.bulk.BulkProcessor;
|
||||
import org.elasticsearch.action.bulk.BulkRequest;
|
||||
import org.elasticsearch.action.bulk.BulkResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.quartz.JobExecutionException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class PopulateElasticsearchJob implements Job {
|
||||
|
||||
public static String ES_CLIENT_KEY = "esClient";
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(PopulateElasticsearchJob.class);
|
||||
|
||||
@Override
|
||||
public void execute(JobExecutionContext context) throws JobExecutionException {
|
||||
final Client esClient = (Client)context.getJobDetail().getJobDataMap().get(ES_CLIENT_KEY);
|
||||
if (Objects.isNull(esClient)) {
|
||||
throw new JobExecutionException(ES_CLIENT_KEY + " is null");
|
||||
}
|
||||
logger.info("Starting PopulateElasticsearchJob");
|
||||
final int processors = Runtime.getRuntime().availableProcessors();
|
||||
final ExecutorService executorService = Executors.newFixedThreadPool(processors * 10);
|
||||
final CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
try {
|
||||
final ResponseHandler<JSONObject> updateCenterHandler = (httpResponse) -> {
|
||||
final StatusLine status = httpResponse.getStatusLine();
|
||||
if (status.getStatusCode() == 200) {
|
||||
final HttpEntity entity = httpResponse.getEntity();
|
||||
final String content = EntityUtils.toString(entity);
|
||||
try {
|
||||
return new JSONObject(String.join("", content.split("\n")[1])).getJSONObject("plugins");
|
||||
} catch (Exception e) {
|
||||
throw new ClientProtocolException("Update center returned invalid JSON");
|
||||
}
|
||||
} else {
|
||||
throw new ClientProtocolException("Unexpected response from update center - " + status.toString());
|
||||
}
|
||||
};
|
||||
final JSONObject plugins = httpClient.execute(new HttpGet("https://updates.jenkins-ci.org/current/update-center.json"), updateCenterHandler);
|
||||
final List<Callable<Void>> tasks = new ArrayList<>();
|
||||
for (String key : plugins.keySet()) {
|
||||
final JSONObject plugin = plugins.getJSONObject(key);
|
||||
final Callable<Void> task = () -> {
|
||||
final ResponseHandler<Void> statsHandler = (httpResponse) -> {
|
||||
final StatusLine status = httpResponse.getStatusLine();
|
||||
if (status.getStatusCode() == 200) {
|
||||
final HttpEntity entity = httpResponse.getEntity();
|
||||
final String content = EntityUtils.toString(entity);
|
||||
try {
|
||||
final JSONObject stats = new JSONObject(content);
|
||||
final JSONObject installations = stats.getJSONObject("installations");
|
||||
final JSONObject installationsPercentage = stats.getJSONObject("installationsPercentage");
|
||||
final JSONObject installationsPerVersion = stats.getJSONObject("installationsPerVersion");
|
||||
final JSONObject installationsPercentagePerVersion = stats.getJSONObject("installationsPercentagePerVersion");
|
||||
plugin.put("installations", new JSONArray(installations.keySet().stream().map((timestamp) -> {
|
||||
final JSONObject installation = new JSONObject();
|
||||
installation.put("timestamp", Long.valueOf(timestamp));
|
||||
installation.put("total", installations.getInt(timestamp));
|
||||
return installation;
|
||||
}).collect(Collectors.toSet())));
|
||||
plugin.put("installationsPercentage", new JSONArray(installationsPercentage.keySet().stream().map((timestamp) -> {
|
||||
final JSONObject installation = new JSONObject();
|
||||
installation.put("timestamp", Long.valueOf(timestamp));
|
||||
installation.put("percentage", installationsPercentage.getDouble(timestamp));
|
||||
return installation;
|
||||
}).collect(Collectors.toSet())));
|
||||
plugin.put("installationsPerVersion", new JSONArray(installationsPerVersion.keySet().stream().map((version) -> {
|
||||
final JSONObject installation = new JSONObject();
|
||||
installation.put("version", version);
|
||||
installation.put("total", installationsPerVersion.getInt(version));
|
||||
return installation;
|
||||
}).collect(Collectors.toSet())));
|
||||
plugin.put("installationsPercentagePerVersion", new JSONArray(installationsPercentagePerVersion.keySet().stream().map((version) -> {
|
||||
final JSONObject installation = new JSONObject();
|
||||
installation.put("version", version);
|
||||
installation.put("total", installationsPercentagePerVersion.getDouble(version));
|
||||
return installation;
|
||||
}).collect(Collectors.toSet())));
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
throw new ClientProtocolException("Problem processing stats", e);
|
||||
}
|
||||
} else {
|
||||
throw new ClientProtocolException("Unexpected response for stats " + status.toString());
|
||||
}
|
||||
};
|
||||
final String url = "http://stats.jenkins-ci.org/plugin-installation-trend/" + plugin.get("name") + ".stats.json";
|
||||
try {
|
||||
httpClient.execute(new HttpGet(url), statsHandler);
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem getting status for plugin " + key);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
tasks.add(task);
|
||||
}
|
||||
executorService.invokeAll(tasks).stream().map(task -> {
|
||||
try {
|
||||
return task.get();
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
});
|
||||
final BulkProcessor bulk = BulkProcessor.builder(esClient, new BulkProcessor.Listener() {
|
||||
|
||||
@Override
|
||||
public void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
logger.info("Indexed " + response.getItems().length + " plugins in " + response.getTook().minutes() + " min " + response.getTook().seconds() + " sec");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
|
||||
}
|
||||
}).build();
|
||||
plugins.keySet().forEach((key) -> {
|
||||
final JSONObject plugin = plugins.getJSONObject(key);
|
||||
bulk.add(esClient.prepareIndex("plugins", "plugins").setId(plugin.getString("name")).setSource(plugin.toString()).request());
|
||||
});
|
||||
bulk.awaitClose(2, TimeUnit.MINUTES);
|
||||
logger.info("Finished PopulateElasticsearchJob");
|
||||
} catch (Exception e) {
|
||||
logger.error("Problem getting plugin information", e);
|
||||
} finally {
|
||||
try {
|
||||
httpClient.close();
|
||||
} catch (IOException e) {
|
||||
logger.error("Problem closing httpClient", e);
|
||||
}
|
||||
if (!executorService.isShutdown()) {
|
||||
executorService.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
{
|
||||
"categories":
|
||||
[
|
||||
{
|
||||
"id": "languagesPlatforms",
|
||||
"title": "Platforms",
|
||||
"description": "Jenkins plugins that are designed to give added support for building, testing or deploying to specific languages or platforms.",
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "ios",
|
||||
"title": "iOS development"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "dotnet",
|
||||
"title": "Azure and .NET"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "android",
|
||||
"title": "Android development"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "ruby",
|
||||
"title": "Ruby development"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "scala",
|
||||
"title": "Scala plugins"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "userInterface",
|
||||
"title": "User interface",
|
||||
"description": "Add-ons designed to make your iteraction with Jenkins better.",
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "ui",
|
||||
"title": "User Interface"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "listview-column",
|
||||
"title": "List view column plugins"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "adminTools",
|
||||
"title": "Administration",
|
||||
"description": "Add functionality to help streamline your administration of Jenkins.",
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "slaves",
|
||||
"title": "Agent launchers and controllers"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "page-decorator",
|
||||
"title": "Page decorators"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "user",
|
||||
"title": "Authentication and user management"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "cluster",
|
||||
"title": "Cluster management and distributed build"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "cli",
|
||||
"title": "CLI extensions"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "buildManagement",
|
||||
"title": "Build management",
|
||||
"description": "Add new means of trigger jobs and new ways to ensure environment consistency for you automated processes.",
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "trigger",
|
||||
"title": "Build triggers"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "buildwrapper",
|
||||
"title": "Build wrappers"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "notifier",
|
||||
"title": "Build notifiers"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "deployment",
|
||||
"title": "Deployment plugins"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "parameter",
|
||||
"title": "Build parameters"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "post-build",
|
||||
"title": "Clean-up actions"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "builder",
|
||||
"title": "Build tools"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "report",
|
||||
"title": "Build reports"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "upload",
|
||||
"title": "Artifact uploaders"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "scm",
|
||||
"title": "Source code management",
|
||||
"description": "Connect to your Jenkins to your SCM.",
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "scm",
|
||||
"title": "Source Code Management"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "scm-related",
|
||||
"title": "Source Code Management related"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": "junk",
|
||||
"hidden": true,
|
||||
"labels":
|
||||
[
|
||||
{
|
||||
"id": "library"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "misc"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "must-be-labeled"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "external"
|
||||
},
|
||||
|
||||
{
|
||||
"id": "maven"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
{
|
||||
"plugins" : {
|
||||
"properties" : {
|
||||
"buildDate" : {
|
||||
"type" : "date",
|
||||
"format" : "MMM dd, YYYY"
|
||||
},
|
||||
"dependencies" : {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"name" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"optional" : {
|
||||
"type" : "boolean"
|
||||
},
|
||||
"version" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
}
|
||||
}
|
||||
},
|
||||
"developers" : {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"developerId" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"email" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"name" : {
|
||||
"type" : "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"exerpt" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"gav" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"installations" : {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"timestamp" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"total" : {
|
||||
"type" : "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"installationsPercentage": {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"timestamp" : {
|
||||
"type" : "long"
|
||||
},
|
||||
"percentage" : {
|
||||
"type" : "double"
|
||||
}
|
||||
}
|
||||
},
|
||||
"installationsPerVersion" : {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"version" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"total" : {
|
||||
"type" : "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"installationsPercentagePerVersion" : {
|
||||
"type" : "nested",
|
||||
"properties" : {
|
||||
"version" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"percentage" : {
|
||||
"type" : "double"
|
||||
}
|
||||
}
|
||||
},
|
||||
"labels" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"name" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"previousTimestamp" : {
|
||||
"type" : "date",
|
||||
"format" : "date_time"
|
||||
},
|
||||
"previousVersion" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"releaseTimestamp" : {
|
||||
"type" : "date",
|
||||
"format" : "date_time"
|
||||
},
|
||||
"requiredCore" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"scm" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"sha1" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"title" : {
|
||||
"type" : "string"
|
||||
},
|
||||
"url" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"version" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
},
|
||||
"wiki" : {
|
||||
"type" : "string",
|
||||
"index" : "not_analyzed"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<configuration>
|
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<root level="debug">
|
||||
<appender-ref ref="STDOUT" />
|
||||
</root>
|
||||
|
||||
<logger name="org.elasticsearch" level="error"/>
|
||||
<logger name="org.quartz" level="error"/>
|
||||
|
||||
</configuration>
|
||||
Loading…
Reference in New Issue