EsvGovernmentBodyOperationOutcomeReaderImpl.java

  1. /*
  2.  * Copyright 2010 James Pether Sörling
  3.  *
  4.  * Licensed under the Apache License, Version 2.0 (the "License");
  5.  * you may not use this file except in compliance with the License.
  6.  * You may obtain a copy of the License at
  7.  *
  8.  *   http://www.apache.org/licenses/LICENSE-2.0
  9.  *
  10.  * Unless required by applicable law or agreed to in writing, software
  11.  * distributed under the License is distributed on an "AS IS" BASIS,
  12.  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13.  * See the License for the specific language governing permissions and
  14.  * limitations under the License.
  15.  *
  16.  *  $Id$
  17.  *  $HeadURL$
  18. */
  19. package com.hack23.cia.service.external.esv.impl;

  20. import java.io.BufferedInputStream;
  21. import java.io.IOException;
  22. import java.io.InputStream;
  23. import java.io.InputStreamReader;
  24. import java.time.Month;
  25. import java.util.ArrayList;
  26. import java.util.Collections;
  27. import java.util.HashMap;
  28. import java.util.List;
  29. import java.util.Map;
  30. import java.util.Map.Entry;
  31. import java.util.Set;
  32. import java.util.stream.Collectors;
  33. import java.util.zip.ZipEntry;
  34. import java.util.zip.ZipInputStream;

  35. import org.apache.commons.codec.Charsets;
  36. import org.apache.commons.csv.CSVFormat;
  37. import org.apache.commons.csv.CSVParser;
  38. import org.apache.commons.csv.CSVRecord;
  39. import org.apache.http.client.fluent.Request;
  40. import org.springframework.beans.factory.annotation.Autowired;
  41. import org.springframework.stereotype.Component;

  42. import com.hack23.cia.service.external.esv.api.GovernmentBodyAnnualOutcomeSummary;
  43. import com.hack23.cia.service.external.esv.api.GovernmentBodyAnnualSummary;

  44. /**
  45.  * The Class EsvGovernmentBodyOperationOutcomeReaderImpl.
  46.  */
  47. @Component
  48. final class EsvGovernmentBodyOperationOutcomeReaderImpl implements EsvGovernmentBodyOperationOutcomeReader {

  49.     /** The Constant ORGANISATIONSNUMMER. */
  50.     private static final String ORGANISATIONSNUMMER = "Organisationsnummer";

  51.     /** The Constant MYNDIGHET. */
  52.     private static final String MYNDIGHET = "Myndighet";

  53.     /** The Constant ÅR. */
  54.     private static final String YEAR = "År";

  55.     /** The Constant UTFALL_DECEMBER. */
  56.     private static final String UTFALL_DECEMBER = "Utfall december";

  57.     /** The Constant UTFALL_NOVEMBER. */
  58.     private static final String UTFALL_NOVEMBER = "Utfall november";

  59.     /** The Constant UTFALL_OKTOBER. */
  60.     private static final String UTFALL_OKTOBER = "Utfall oktober";

  61.     /** The Constant UTFALL_SEPTEMBER. */
  62.     private static final String UTFALL_SEPTEMBER = "Utfall september";

  63.     /** The Constant UTFALL_AUGUSTI. */
  64.     private static final String UTFALL_AUGUSTI = "Utfall augusti";

  65.     /** The Constant UTFALL_JULI. */
  66.     private static final String UTFALL_JULI = "Utfall juli";

  67.     /** The Constant UTFALL_JUNI. */
  68.     private static final String UTFALL_JUNI = "Utfall juni";

  69.     /** The Constant UTFALL_MAJ. */
  70.     private static final String UTFALL_MAJ = "Utfall maj";

  71.     /** The Constant UTFALL_APRIL. */
  72.     private static final String UTFALL_APRIL = "Utfall april";

  73.     /** The Constant UTFALL_MARS. */
  74.     private static final String UTFALL_MARS = "Utfall mars";

  75.     /** The Constant UTFALL_FEBRUARI. */
  76.     private static final String UTFALL_FEBRUARI = "Utfall februari";

  77.     /** The Constant UTFALL_JANUARI. */
  78.     private static final String UTFALL_JANUARI = "Utfall januari";

  79.     /** The Constant SPECIFIC_OUTGOING_FIELDS. */
  80.     private static final String[] SPECIFIC_OUTGOING_FIELDS = new String[] { "Inkomsttyp", "Inkomsttypsnamn", "Inkomsthuvudgrupp", "Inkomsthuvudgruppsnamn", "Inkomsttitelgrupp", "Inkomsttitelgruppsnamn", "Inkomsttitel", "Inkomsttitelsnamn", "Inkomstundertitel", "Inkomstundertitelsnamn"};

  81.     /** The Constant SPECIFIC_INCOMING_FIELDS. */
  82.     private static final String[] SPECIFIC_INCOMING_FIELDS = new String[] { "Utgiftsområde", "Utgiftsområdesnamn", "Anslag", "Anslagsnamn", "Anslagspost", "Anslagspostsnamn", "Anslagsdelpost", "Anslagsdelpostsnamn"};

  83.     /** The esv excel reader. */
  84.     @Autowired
  85.     private EsvExcelReader esvExcelReader;
  86.    
  87.     private List<GovernmentBodyAnnualOutcomeSummary> incomeCsvValues;

  88.     private List<GovernmentBodyAnnualOutcomeSummary> outGoingCsvValues;

  89.     /**
  90.      * Instantiates a new esv government body operation outcome reader impl.
  91.      */
  92.     public EsvGovernmentBodyOperationOutcomeReaderImpl() {
  93.         super();
  94.     }

  95.     @Override
  96.     public synchronized List<GovernmentBodyAnnualOutcomeSummary> readIncomeCsv() throws IOException {
  97.         if (incomeCsvValues == null) {
  98.             incomeCsvValues = readUsingZipInputStream(Request.Get(
  99.                 "https://www.esv.se/psidata/manadsutfall/GetFile/?documentType=Inkomst&fileType=Zip&fileName=M%C3%A5nadsutfall%20inkomster%20januari%202006%20-%20augusti%202018,%20definitivt.zip&year=2018&month=8&status=Definitiv")
  100.                 .execute().returnContent().asStream(),SPECIFIC_OUTGOING_FIELDS);
  101.         }
  102.         return Collections.unmodifiableList(incomeCsvValues);
  103.     }
  104.    
  105.     @Override
  106.     public synchronized List<GovernmentBodyAnnualOutcomeSummary> readOutgoingCsv() throws IOException {    
  107.         if (outGoingCsvValues == null) {
  108.             outGoingCsvValues = readUsingZipInputStream(Request.Get(
  109.                 "https://www.esv.se/psidata/manadsutfall/GetFile/?documentType=Utgift&fileType=Zip&fileName=M%C3%A5nadsutfall%20utgifter%20januari%202006%20-%20augusti%202018,%20definitivt.zip&year=2018&month=8&status=Definitiv")
  110.                 .execute().returnContent().asStream(),SPECIFIC_INCOMING_FIELDS);
  111.         }
  112.         return Collections.unmodifiableList(outGoingCsvValues);
  113.     }

  114.     /**
  115.      * Read using zip input stream.
  116.      *
  117.      * @param inputStream
  118.      *            the input stream
  119.      * @param specificFields
  120.      *            the specific fields
  121.      * @return the list
  122.      * @throws IOException
  123.      *             Signals that an I/O exception has occurred.
  124.      */
  125.     private List<GovernmentBodyAnnualOutcomeSummary> readUsingZipInputStream(final InputStream inputStream,final String[] specificFields) throws IOException {      
  126.         final BufferedInputStream bis = new BufferedInputStream(inputStream);
  127.         final ZipInputStream is = new ZipInputStream(bis);

  128.         final List<GovernmentBodyAnnualOutcomeSummary> list = new ArrayList<>();
  129.         try {
  130.             ZipEntry entry;
  131.             while ((entry = is.getNextEntry()) != null) {
  132.                 list.addAll(readCsvContent(is,specificFields));
  133.             }
  134.         } finally {
  135.             is.close();
  136.         }
  137.         return list;
  138.     }

  139.     /**
  140.      * Read csv content.
  141.      *
  142.      * @param is
  143.      *            the is
  144.      * @param specificFields
  145.      *            the specific fields
  146.      * @return the list
  147.      * @throws IOException
  148.      *             Signals that an I/O exception has occurred.
  149.      */
  150.     private List<GovernmentBodyAnnualOutcomeSummary> readCsvContent(final InputStream is,final String[] specificFields) throws IOException {
  151.         final CSVParser parser = CSVParser.parse(new InputStreamReader(is,Charsets.UTF_8), CSVFormat.EXCEL.withHeader().withDelimiter(';'));
  152.         final List<CSVRecord> records = parser.getRecords();
  153.         records.remove(0);
  154.        
  155.         Map<Integer, Map<String,String>> orgMinistryMap = createOrgMinistryMap(esvExcelReader.getDataPerMinistry(null));
  156.        
  157.         final List<GovernmentBodyAnnualOutcomeSummary> list = new ArrayList<>();
  158.        
  159.         for (final CSVRecord csvRecord : records) {
  160.             final GovernmentBodyAnnualOutcomeSummary governmentBodyAnnualOutcomeSummary = new GovernmentBodyAnnualOutcomeSummary(csvRecord.get(MYNDIGHET), csvRecord.get(ORGANISATIONSNUMMER), orgMinistryMap.get(Integer.parseInt(csvRecord.get(YEAR))).get(csvRecord.get(ORGANISATIONSNUMMER).replaceAll("-", "")), Integer.parseInt(csvRecord.get(YEAR)));
  161.            
  162.             for (final String field : specificFields) {            
  163.                 governmentBodyAnnualOutcomeSummary.addDescriptionField(field,csvRecord.get(field));
  164.             }
  165.            
  166.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.JANUARY.getValue(),csvRecord.get(UTFALL_JANUARI));
  167.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.FEBRUARY.getValue(),csvRecord.get(UTFALL_FEBRUARI));
  168.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.MARCH.getValue(),csvRecord.get(UTFALL_MARS));
  169.            
  170.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.APRIL.getValue(),csvRecord.get(UTFALL_APRIL));
  171.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.MAY.getValue(),csvRecord.get(UTFALL_MAJ));
  172.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.JUNE.getValue(),csvRecord.get(UTFALL_JUNI));
  173.            
  174.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.JULY.getValue(),csvRecord.get(UTFALL_JULI));
  175.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.AUGUST.getValue(),csvRecord.get(UTFALL_AUGUSTI));
  176.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.SEPTEMBER.getValue(),csvRecord.get(UTFALL_SEPTEMBER));
  177.            
  178.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.OCTOBER.getValue(),csvRecord.get(UTFALL_OKTOBER));
  179.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.NOVEMBER.getValue(),csvRecord.get(UTFALL_NOVEMBER));
  180.             addResultForMonth(governmentBodyAnnualOutcomeSummary,Month.DECEMBER.getValue(),csvRecord.get(UTFALL_DECEMBER));
  181.            
  182.             list.add(governmentBodyAnnualOutcomeSummary);
  183.         }
  184.        
  185.         return list;
  186.     }

  187.     /**
  188.      * Creates the org ministry map.
  189.      *
  190.      * @param data the data
  191.      * @return the map
  192.      */
  193.     private static Map<Integer, Map<String, String>> createOrgMinistryMap(
  194.             Map<Integer, List<GovernmentBodyAnnualSummary>> data) {
  195.         Map<Integer, Map<String,String>> orgMinistryMap = new HashMap<>();
  196.        
  197.         Set<Entry<Integer, List<GovernmentBodyAnnualSummary>>> entrySet = data.entrySet();
  198.        
  199.         for (Entry<Integer, List<GovernmentBodyAnnualSummary>> entry : entrySet) {      
  200.             orgMinistryMap.put(entry.getKey(), entry.getValue().stream().collect(Collectors.groupingBy(t -> t.getOrgNumber().replaceAll("-","") ,Collectors.collectingAndThen(
  201.                     Collectors.toList(),
  202.                     values -> values.get(0).getMinistry()))));
  203.         }
  204.        
  205.         return orgMinistryMap;
  206.     }

  207.     /**
  208.      * Adds the result for month.
  209.      *
  210.      * @param governmentBodyAnnualOutcomeSummary
  211.      *            the government body annual outcome summary
  212.      * @param month
  213.      *            the month
  214.      * @param value
  215.      *            the value
  216.      */
  217.     private static void addResultForMonth(final GovernmentBodyAnnualOutcomeSummary governmentBodyAnnualOutcomeSummary, final int month,
  218.             final String value) {
  219.         if (value != null && value.length() >0 ) {
  220.             governmentBodyAnnualOutcomeSummary.addData(month,Double.valueOf(value.replaceAll(",", ".")));
  221.         }
  222.     }

  223. }