Implement ConfigurationService

Extracts getting plugin data into a separate service so it can be mocked
in tests and it's just generally good practice.

README.asc

@@ -26,7 +26,7 @@ of the application.
 == Run Local Plugin Site API
 
 ----
-mvn jetty:run
+DATA_FILE_URL="http://url.to/plugins.json.gzip" mvn jetty:run
 ----
 
 This will launch an embedded Jetty container accessible at `http://localhost:8080`.
@@ -35,7 +35,7 @@ This will launch an embedded Jetty container accessible at `http://localhost:808
 
 ----
 docker build -t jenkinsciinfra/plugin-site-api .
-docker run -p 8080:8080 -it jenkinsciinfra/plugin-site-api
+docker run -p 8080:8080 -it -e DATA_FILE_URL="http://url.to/plugins.json.gzip" jenkinsciinfra/plugin-site-api
 ----
 
 == Rebuild Elasticsearch data
@@ -44,8 +44,8 @@ docker run -p 8080:8080 -it jenkinsciinfra/plugin-site-api
 mvn -P generatePluginData
 ----
 
-This will generate a new file in `src/main/resources/elasticsearch/data/plugins.json.gzip`
-consisting of plugin information and installation statistics.
+This will generate a new file in `target/data/plugins.json.gzip` consisting of plugin information and installation
+statistics. This file should be uploaded to DATA_FILE_URL.
 
 == REST API Reference
 

pom.xml

@@ -49,6 +49,11 @@
       <artifactId>guava</artifactId>
       <version>19.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.4</version>
+    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>

src/main/java/io/jenkins/plugins/datastore/EmbeddedElasticsearchServer.java

@@ -2,13 +2,8 @@ package io.jenkins.plugins.datastore;
 
 import io.jenkins.plugins.commons.JsonObjectMapper;
 import io.jenkins.plugins.models.Plugin;
+import io.jenkins.plugins.services.ConfigurationService;
 import org.apache.commons.io.FileUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -24,12 +19,12 @@ import org.slf4j.LoggerFactory;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
-import java.io.*;
+import javax.inject.Inject;
+import java.io.File;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.stream.Collectors;
-import java.util.zip.GZIPInputStream;
 
 /**
  * Copied and modified from:
@@ -46,6 +41,9 @@ public class EmbeddedElasticsearchServer {
     return node.client();
   }
 
+  @Inject
+  private ConfigurationService configurationService;
+
   @PostConstruct
   public void postConstruct() {
     logger.info("Initialize elasticsearch");
@@ -75,41 +73,15 @@ public class EmbeddedElasticsearchServer {
 
   private void createAndPopulateIndex() {
     try {
-      final String dataFile = retrieveDataFile();
-      reIndex(dataFile);
+      final String data = configurationService.getIndexData();
+      reIndex(data);
     } catch (Exception e) {
       logger.error("Problem creating and populating index", e);
       throw new RuntimeException("Problem creating and populating index", e);
     }
   }
 
-  private String retrieveDataFile() {
-    final CloseableHttpClient httpClient = HttpClients.createDefault();
-    try {
-      final String url = "http://localhost:8089/plugins.json.gzip";
-      final HttpGet get = new HttpGet(url);
-      final CloseableHttpResponse response = httpClient.execute(get);
-      if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
-        final HttpEntity entity = response.getEntity();
-        final InputStream inputStream = entity.getContent();
-        final File dataFile = File.createTempFile("plugins", ".json.gzip");
-        FileUtils.copyToFile(inputStream, dataFile);
-        return readGzipFile(dataFile);
-      } else {
-        logger.error("Data file not found");
-        throw new RuntimeException("Data file not found");
-      }
-    } catch (Exception e) {
-      logger.error("Problem getting data file", e);
-      throw new RuntimeException("Problem getting data file", e);
-    } finally {
-      try {
-        httpClient.close();
-      } catch (IOException e) {
-        logger.warn("Problem closing HttpClient", e);
-      }
-    }
-  }
+
 
   private void reIndex(String data) {
     final ClassLoader cl = getClass().getClassLoader();
@@ -149,13 +121,6 @@ public class EmbeddedElasticsearchServer {
     }
   }
 
-  private String readGzipFile(final File file) {
-    try(final BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), "utf-8"))) {
-      return reader.lines().collect(Collectors.joining());
-    } catch (Exception e) {
-      logger.error("Problem decompressing plugin data", e);
-      throw new RuntimeException("Problem decompressing plugin data", e);
-    }
-  }
+
 
 }

src/main/java/io/jenkins/plugins/services/Binder.java

@@ -1,5 +1,6 @@
 package io.jenkins.plugins.services;
 
+import io.jenkins.plugins.services.impl.DefaultConfigurationService;
 import io.jenkins.plugins.services.impl.ElasticsearchDatastoreService;
 import io.jenkins.plugins.services.impl.HttpClientWikiService;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -11,6 +12,7 @@ import javax.inject.Singleton;
  *
  * <p>Binds</p>
  * <ul>
+ *   <li><code>DefaultConfigurationService</code> to <code>ConfigurationService</code>  as a <code>Singleton</code></li>
  *   <li><code>ElasticsearchDatastoreService</code> to <code>DatastoreService</code> as a <code>Singleton</code></li>
  *   <li><code>HttpClientWikiService</code> to <code>WikiService</code> as a <code>Singleton</code></li>
  * </ul>
@@ -22,6 +24,7 @@ public class Binder extends AbstractBinder {
 
   @Override
   protected void configure() {
+    bind(DefaultConfigurationService.class).to(ConfigurationService.class).in(Singleton.class);
     bind(ElasticsearchDatastoreService.class).to(DatastoreService.class).in(Singleton.class);
     bind(HttpClientWikiService.class).to(WikiService.class).in(Singleton.class);
   }

src/main/java/io/jenkins/plugins/services/ConfigurationService.java

@@ -0,0 +1,16 @@
+package io.jenkins.plugins.services;
+
+/**
+ * <p>Get various configuration pieces for the application</p>
+ */
+public interface ConfigurationService {
+
+  /**
+   * <p>Get index data need to populating Elasticsearch</p>
+   *
+   * @return JSON content
+   * @throws ServiceException in case something goes wrong
+     */
+  String getIndexData() throws ServiceException;
+
+}

src/main/java/io/jenkins/plugins/services/impl/DefaultConfigurationService.java

@@ -0,0 +1,81 @@
+package io.jenkins.plugins.services.impl;
+
+import io.jenkins.plugins.services.ConfigurationService;
+import io.jenkins.plugins.services.ServiceException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * <p>Default implementation of <code>ConfigurationService</code></p>
+ */
+public class DefaultConfigurationService implements ConfigurationService {
+
+  private final Logger logger = LoggerFactory.getLogger(DefaultConfigurationService.class);
+
+  @Override
+  public String getIndexData() throws ServiceException {
+    final CloseableHttpClient httpClient = HttpClients.createDefault();
+    try {
+      final String url = getDataFileUrl();
+      final HttpGet get = new HttpGet(url);
+      final CloseableHttpResponse response = httpClient.execute(get);
+      if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+        final HttpEntity entity = response.getEntity();
+        final InputStream inputStream = entity.getContent();
+        final File dataFile = File.createTempFile("plugins", ".json.gzip");
+        FileUtils.copyToFile(inputStream, dataFile);
+        return readGzipFile(dataFile);
+      } else {
+        logger.error("Data file not found");
+        throw new RuntimeException("Data file not found");
+      }
+    } catch (Exception e) {
+      logger.error("Problem getting data file", e);
+      throw new ServiceException("Problem getting data file", e);
+    } finally {
+      try {
+        httpClient.close();
+      } catch (IOException e) {
+        logger.warn("Problem closing HttpClient", e);
+      }
+    }
+  }
+
+  private String getDataFileUrl() {
+    if (System.getenv().containsKey("DATA_FILE_URL")) {
+      final String url = StringUtils.trimToNull(System.getenv("DATA_FILE_URL"));
+      if (url == null) {
+        throw new RuntimeException("Environment variable 'DATA_FILE_URL' is empty");
+      }
+      return url;
+    } else {
+      final String url = StringUtils.trimToNull(System.getProperty("data.file.url"));
+      if (url == null) {
+        throw new RuntimeException("System property 'data.file.url' is not given");
+      }
+      return url;
+    }
+  }
+
+  private String readGzipFile(final File file) {
+    try(final BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), "utf-8"))) {
+      return reader.lines().collect(Collectors.joining());
+    } catch (Exception e) {
+      logger.error("Problem decompressing plugin data", e);
+      throw new RuntimeException("Problem decompressing plugin data", e);
+    }
+  }
+
+}

src/test/java/io/jenkins/plugins/services/DatastoreServiceIntegrationTest.java

@@ -1,13 +1,17 @@
 package io.jenkins.plugins.services;
 
 import io.jenkins.plugins.models.*;
+import io.jenkins.plugins.services.impl.ElasticsearchDatastoreService;
+import io.jenkins.plugins.services.impl.HttpClientWikiService;
 import org.glassfish.hk2.api.ServiceLocator;
 import org.glassfish.hk2.utilities.ServiceLocatorUtilities;
+import org.glassfish.hk2.utilities.binding.AbstractBinder;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import javax.inject.Singleton;
 import java.util.Arrays;
 import java.util.Collections;
 
@@ -22,7 +26,14 @@ public class DatastoreServiceIntegrationTest {
   public static void setUp() throws Exception {
     locator  = ServiceLocatorUtilities.bind(
       new io.jenkins.plugins.datastore.Binder(),
-      new io.jenkins.plugins.services.Binder());
+      new AbstractBinder() {
+        @Override
+        protected void configure() {
+          bind(MockConfigurationService.class).to(ConfigurationService.class).in(Singleton.class);
+          bind(ElasticsearchDatastoreService.class).to(DatastoreService.class).in(Singleton.class);
+          bind(HttpClientWikiService.class).to(WikiService.class).in(Singleton.class);
+        }
+      });
     datastoreService = locator.getService(DatastoreService.class);
   }
 

src/test/java/io/jenkins/plugins/services/MockConfigurationService.java

@@ -0,0 +1,24 @@
+package io.jenkins.plugins.services;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class MockConfigurationService implements ConfigurationService {
+
+  private final Logger logger = LoggerFactory.getLogger(MockConfigurationService.class);
+
+  @Override
+  public String getIndexData() throws ServiceException {
+    try {
+      logger.info("Using test plugin data");
+      final ClassLoader cl = getClass().getClassLoader();
+      final File mappingFile = new File(cl.getResource("plugins.json").getFile());
+      return FileUtils.readFileToString(mappingFile, "utf-8");
+    } catch (Exception e) {
+      throw new RuntimeException("Can't get test plugin data");
+    }
+  }
+}