WorldbankIndicatorApiImpl.java
- /*
- * Copyright 2010 James Pether Sörling
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * $Id$
- * $HeadURL$
- */
- package com.hack23.cia.service.external.worldbank.impl;
- import java.io.BufferedInputStream;
- import java.io.BufferedReader;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.InputStreamReader;
- import java.util.ArrayList;
- import java.util.Collections;
- import java.util.List;
- import java.util.zip.ZipEntry;
- import java.util.zip.ZipInputStream;
- import org.apache.commons.codec.Charsets;
- import org.apache.commons.csv.CSVFormat;
- import org.apache.commons.csv.CSVParser;
- import org.apache.commons.csv.CSVRecord;
- import org.apache.http.client.fluent.Request;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.springframework.beans.factory.annotation.Autowired;
- import org.springframework.beans.factory.annotation.Qualifier;
- import org.springframework.oxm.Unmarshaller;
- import org.springframework.stereotype.Component;
- import com.hack23.cia.model.external.worldbank.indicators.impl.IndicatorElement;
- import com.hack23.cia.model.external.worldbank.indicators.impl.IndicatorsElement;
- import com.hack23.cia.service.external.common.api.XmlAgent;
- import com.hack23.cia.service.external.common.api.XmlAgentException;
- import com.hack23.cia.service.external.worldbank.api.DataFailureException;
- import com.hack23.cia.service.external.worldbank.api.WorldBankIndicatorApi;
- /**
- * The Class WorldbankIndicatorApiImpl.
- */
- @Component
- final class WorldbankIndicatorApiImpl extends BaseWorldBankApiImpl implements WorldBankIndicatorApi {
- private static final int IGNORE_TOP_HEADERS_LINE = 4;
- /** The Constant LOGGER. */
- private static final Logger LOGGER = LoggerFactory.getLogger(WorldbankIndicatorApiImpl.class);
- /** The Constant INDICATORS. */
- private static final String INDICATORS = "https://api.worldbank.org/indicators?per_page=5000";
- private static final String PAGE_NUMBER = "&page=";
- /** The Constant PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST. */
- private static final String PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST = "Problem getting worldbank indicator list";
- private static final int SECOND_PAGE = 2;
- /**
- * The Constant
- * XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL.
- */
- private static final String XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL = "xmlns:wb=\"http://indicators.worldbank.external.model.cia.hack23.com/impl\"";
- /** The indicators unmarshaller. */
- @Autowired
- @Qualifier("worldbankOrgIndicatorsMarshaller")
- private Unmarshaller indicatorsUnmarshaller;
- /**
- * Instantiates a new worldbank indicator api impl.
- */
- @Autowired
- public WorldbankIndicatorApiImpl(final XmlAgent xmlAgent) {
- super(xmlAgent);
- }
- @Override
- public List<IndicatorElement> getIndicators() throws DataFailureException {
- final List<IndicatorElement> result = new ArrayList<>();
- try {
- final IndicatorsElement firstPage = (IndicatorsElement) getXmlAgent().unmarshallXml(indicatorsUnmarshaller,
- INDICATORS, null, XMLNS_WB_HTTP_WWW_WORLDBANK_ORG,
- XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL);
- result.addAll(firstPage.getIndicator());
- for (int pageNumber = SECOND_PAGE; pageNumber < firstPage.getPages().intValue(); pageNumber++) {
- final IndicatorsElement otherPageResult = (IndicatorsElement) getXmlAgent().unmarshallXml(indicatorsUnmarshaller,
- INDICATORS + PAGE_NUMBER + pageNumber, null, XMLNS_WB_HTTP_WWW_WORLDBANK_ORG,
- XMLNS_WB_HTTP_INDICATORS_WORLDBANK_EXTERNAL_MODEL_CIA_HACK23_COM_IMPL);
- result.addAll(otherPageResult.getIndicator());
- }
- } catch (final XmlAgentException e) {
- LOGGER.warn(PROBLEM_GETTING_WORLDBANK_INDICATOR_LIST);
- throw new DataFailureException(e);
- }
- return result;
- }
- @Override
- public List<String> getIndicatorsWithSwedishData() throws DataFailureException {
- try {
- return Collections.unmodifiableList(readUsingZipInputStream(Request.Get("http://api.worldbank.org/v2/en/country/SWE?downloadformat=csv").execute().returnContent().asStream()));
- } catch (final IOException e) {
- throw new DataFailureException(e);
- }
- }
- /**
- * Read using zip input stream.
- *
- * @param inputStream
- * the input stream
- * @param specificFields
- * the specific fields
- * @return the list
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- private static List<String> readUsingZipInputStream(final InputStream inputStream) throws IOException {
- final BufferedInputStream bis = new BufferedInputStream(inputStream);
- final ZipInputStream is = new ZipInputStream(bis);
- final List<String> list = new ArrayList<>();
- try {
- ZipEntry entry;
-
- while ((entry = is.getNextEntry()) != null) {
- if (entry.getName().startsWith("API_SWE_")) {
- list.addAll(readCsvContent(is));
- }
- }
- } finally {
- is.close();
- }
- return list;
- }
-
- /**
- * Read csv content.
- *
- * @param is
- * the is
- * @return the list
- * @throws IOException
- * Signals that an I/O exception has occurred.
- */
- private static List<String> readCsvContent(final InputStream is) throws IOException {
- final BufferedReader reader = new BufferedReader(new InputStreamReader(is,Charsets.UTF_8));
- for (int i = 0; i < IGNORE_TOP_HEADERS_LINE; i++) {
- String ignoreFirstLinesWithHeaders = reader.readLine();
- }
-
- final CSVParser parser = CSVParser.parse(reader, CSVFormat.EXCEL.withHeader().withDelimiter(','));
- final List<CSVRecord> records = parser.getRecords();
- records.remove(0);
-
- final List<String> list = new ArrayList<>();
-
- for (final CSVRecord csvRecord : records) {
- list.add(csvRecord.get("Indicator Code"));
- }
-
- return list;
- }
-
- }