WorldbankIndicatorApiImpl.java

  1. /*
  2.  * Copyright 2010 James Pether Sörling
  3.  *
  4.  * Licensed under the Apache License, Version 2.0 (the "License");
  5.  * you may not use this file except in compliance with the License.
  6.  * You may obtain a copy of the License at
  7.  *
  8.  *   http://www.apache.org/licenses/LICENSE-2.0
  9.  *
  10.  * Unless required by applicable law or agreed to in writing, software
  11.  * distributed under the License is distributed on an "AS IS" BASIS,
  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13.  * See the License for the specific language governing permissions and
  14.  * limitations under the License.
  15.  *
  16.  *  $Id$
  17.  *  $HeadURL$
  18. */
  19. package com.hack23.cia.service.external.worldbank.impl;

  20. import java.io.BufferedInputStream;
  21. import java.io.BufferedReader;
  22. import java.io.IOException;
  23. import java.io.InputStream;
  24. import java.io.InputStreamReader;
  25. import java.util.ArrayList;
  26. import java.util.Collections;
  27. import java.util.List;
  28. import java.util.zip.ZipEntry;
  29. import java.util.zip.ZipInputStream;

  30. import org.apache.commons.codec.Charsets;
  31. import org.apache.commons.csv.CSVFormat;
  32. import org.apache.commons.csv.CSVParser;
  33. import org.apache.commons.csv.CSVRecord;
  34. import org.apache.http.client.fluent.Request;
  35. import org.slf4j.Logger;
  36. import org.slf4j.LoggerFactory;
  37. import org.springframework.beans.factory.annotation.Autowired;
  38. import org.springframework.beans.factory.annotation.Qualifier;
  39. import org.springframework.oxm.Unmarshaller;
  40. import org.springframework.stereotype.Component;

  41. import com.hack23.cia.model.external.worldbank.indicators.impl.IndicatorElement;
  42. import com.hack23.cia.model.external.worldbank.indicators.impl.IndicatorsElement;
  43. import com.hack23.cia.service.external.common.api.XmlAgent;
  44. import com.hack23.cia.service.external.common.api.XmlAgentException;
  45. import com.hack23.cia.service.external.worldbank.api.DataFailureException;
  46. import com.hack23.cia.service.external.worldbank.api.WorldBankIndicatorApi;

  47. /**
  48.  * The Class WorldbankIndicatorApiImpl.
  49.  */
  50. @Component
  51. final class WorldbankIndicatorApiImpl extends BaseWorldBankApiImpl implements WorldBankIndicatorApi {

  52.     private static final int IGNORE_TOP_HEADERS_LINE = 4;

  53.     /** The Constant LOGGER. */
  54.     private static final Logger LOGGER = LoggerFactory.getLogger(WorldbankIndicatorApiImpl.class);

  55.     /** The Constant INDICATORS. */
  56.     private static final String INDICATORS = "https://api.worldbank.org/indicators?per_page=5000";

  57.     private static final String PAGE_NUMBER = "&page=";

  58.     /** The Constant PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST. */
  59.     private static final String PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST = "Problem getting worldbank indicator list";

  60.     private static final int SECOND_PAGE = 2;

  61.     /**
  62.      * The Constant
  63.      * XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL.
  64.      */
  65.     private static final String XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL = "xmlns:wb=\"http://indicators.worldbank.external.model.cia.hack23.com/impl\"";


  66.     /** The indicators unmarshaller. */
  67.     @Autowired
  68.     @Qualifier("worldbankOrgIndicatorsMarshaller")
  69.     private Unmarshaller indicatorsUnmarshaller;

  70.     /**
  71.      * Instantiates a new worldbank indicator api impl.
  72.      */
  73.     @Autowired
  74.     public WorldbankIndicatorApiImpl(final XmlAgent xmlAgent) {
  75.         super(xmlAgent);
  76.     }

  77.     @Override
  78.     public List<IndicatorElement> getIndicators() throws DataFailureException {
  79.         final List<IndicatorElement> result = new ArrayList<>();

  80.         try {
  81.             final IndicatorsElement firstPage = (IndicatorsElement) getXmlAgent().unmarshallXml(indicatorsUnmarshaller,
  82.                     INDICATORS, null, XMLNS_WB_HTTP_WWW_WORLDBANK_ORG,
  83.                     XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL);
  84.             result.addAll(firstPage.getIndicator());

  85.             for (int pageNumber = SECOND_PAGE; pageNumber < firstPage.getPages().intValue(); pageNumber++) {
  86.                 final IndicatorsElement otherPageResult = (IndicatorsElement) getXmlAgent().unmarshallXml(indicatorsUnmarshaller,
  87.                         INDICATORS + PAGE_NUMBER + pageNumber, null, XMLNS_WB_HTTP_WWW_WORLDBANK_ORG,
  88.                         XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL);
  89.                 result.addAll(otherPageResult.getIndicator());
  90.             }

  91.         } catch (final XmlAgentException e) {
  92.             LOGGER.warn(PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST);
  93.             throw new DataFailureException(e);
  94.         }

  95.         return result;
  96.     }

  97.     @Override
  98.     public List<String> getIndicatorsWithSwedishData() throws DataFailureException {        
  99.         try {
  100.             return Collections.unmodifiableList(readUsingZipInputStream(Request.Get("http://api.worldbank.org/v2/en/country/SWE?downloadformat=csv").execute().returnContent().asStream()));
  101.         } catch (final IOException e) {
  102.             throw new DataFailureException(e);
  103.         }
  104.     }

  105.     /**
  106.      * Read using zip input stream.
  107.      *
  108.      * @param inputStream
  109.      *            the input stream
  110.      * @param specificFields
  111.      *            the specific fields
  112.      * @return the list
  113.      * @throws IOException
  114.      *             Signals that an I/O exception has occurred.
  115.      */
  116.     private static List<String> readUsingZipInputStream(final InputStream inputStream) throws IOException {    
  117.         final BufferedInputStream bis = new BufferedInputStream(inputStream);
  118.         final ZipInputStream is = new ZipInputStream(bis);

  119.         final List<String> list = new ArrayList<>();
  120.         try {
  121.             ZipEntry entry;
  122.            
  123.             while ((entry = is.getNextEntry()) != null) {
  124.                 if (entry.getName().startsWith("API_SWE_")) {                  
  125.                     list.addAll(readCsvContent(is));                    
  126.                 }                      
  127.             }
  128.         } finally {
  129.             is.close();
  130.         }
  131.         return list;
  132.     }
  133.    
  134.     /**
  135.      * Read csv content.
  136.      *
  137.      * @param is
  138.      *            the is
  139.      * @return the list
  140.      * @throws IOException
  141.      *             Signals that an I/O exception has occurred.
  142.      */
  143.     private static List<String> readCsvContent(final InputStream is) throws IOException {      
  144.         final BufferedReader reader = new BufferedReader(new InputStreamReader(is,Charsets.UTF_8));    
  145.         for (int i = 0; i < IGNORE_TOP_HEADERS_LINE; i++) {
  146.             String ignoreFirstLinesWithHeaders = reader.readLine();        
  147.         }
  148.        
  149.         final CSVParser parser = CSVParser.parse(reader, CSVFormat.EXCEL.withHeader().withDelimiter(','));
  150.         final List<CSVRecord> records = parser.getRecords();
  151.         records.remove(0);
  152.        
  153.         final List<String> list = new ArrayList<>();
  154.        
  155.         for (final CSVRecord csvRecord : records) {        
  156.             list.add(csvRecord.get("Indicator Code"));
  157.         }
  158.        
  159.         return list;
  160.     }

  161.    

  162. }