GeoServiceImpl.java

/*
 * Copyright 2018 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.service.impl;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.Country;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.model.impl.ITPGRFAStatus;
import org.genesys.server.model.impl.QCountry;
import org.genesys.server.model.vocab.VocabularyTerm;
import org.genesys.server.persistence.CountryRepository;
import org.genesys.server.persistence.ITPGRFAStatusRepository;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.CRMException;
import org.genesys.server.service.ContentService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.GeoService;
import org.genesys.server.service.InstituteService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.service.filter.CountryFilter;
import org.genesys.server.service.worker.CountryInfo;
import org.genesys.server.service.worker.CustomISO3166Source;
import org.genesys.server.service.worker.CustomISO3166Source.Custom3166Entry;
import org.genesys.server.service.worker.DavrosCountrySource;
import org.genesys.server.service.worker.GeoNamesCountrySource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Sets;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.impl.JPAQueryFactory;

@Service
@Transactional(readOnly = true)
public class GeoServiceImpl implements GeoService {
    public static final Logger LOG = LoggerFactory.getLogger(GeoServiceImpl.class);

    @Autowired
    private JPAQueryFactory jpaQueryFactory;

    @Autowired
    CountryRepository countryRepository;

    @Autowired
    ContentService contentService;

    @Autowired
    ITPGRFAStatusRepository itpgrfaRepository;

    @Autowired(required = false)
    private GeoNamesCountrySource geoNamesCountrySource;

    @Autowired(required = false)
    private DavrosCountrySource davrosCountrySource;

    @Autowired(required = false)
    private CustomISO3166Source customISO3166Source;

    @Autowired(required = false)
    private ElasticsearchService elasticsearchService;

    @Autowired
    private InstituteService instituteService;

    @Autowired
    private AccessionService accessionService;

    @Autowired
    private GenesysService genesysService;

    private final ObjectMapper mapper = new ObjectMapper();

    @Override
    public List<Country> listAll() {
        return countryRepository.findAll(Sort.by("name", "current"));
    }

    @Override
    public List<Country> listAll(final Locale locale) {
        final List<Country> all = listAll();
        Collections.sort(all, new Comparator<Country>() {
            @Override
            public int compare(Country o1, Country o2) {
                return o1.getName(locale).compareTo(o2.getName(locale));
            }
        });
        return all;
    }

    @Override
    public List<Country> listActive(final Locale locale) {
        final List<Country> countries = countryRepository.findByCurrent(true);
        Country.sort(countries, locale);
        return countries;
    }

    @Override
    public long countActive() {
        return countryRepository.countByCurrent(true);
    }

    @Override
    public List<Country> listITPGRFA(Locale locale) {
        final List<Country> countries = countryRepository.findITPGRFA();
        Country.sort(countries, locale);
        return countries;
    }

    @Override
    public Country getCountry(String isoCode) {
        Country country = null;
        if (isoCode.length() == 3) {
            country = countryRepository.findByCode3(isoCode);
        } else if (isoCode.length() == 2) {
            // RO = ROU and ROM
            country = countryRepository.findByCode2AndCurrent(isoCode, true);
            if (country == null) {
                country = countryRepository.findByCode2AndCurrent(isoCode, false);
            }
        }

        return country;
    }

    @Override
    public Country findCountry(String countryString) {
        Country country = getCountry(countryString);

        // Let's try the name
        if (country == null) {
            country = countryRepository.findByName(countryString);
        }

        // Let's try translations
        if (country == null) {
            country = findCountryByName(countryString);
        }
        
        // Let's sanitize input
        if (country == null) {
            country = findCountryByName(countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
            if (country != null) {
                LOG.warn("Found country {} for {}", country.getName(), countryString);
            } else {
                LOG.warn("No country for {} - queried by {}", countryString, countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
            }
        }

        return country;
    }

    /**
     * Check if we have a country that has
     *
     * @param name in i18n
     * @return
     */
    private Country findCountryByName(String name) {
        final List<Country> countries = countryRepository.findWithI18N("%" + name.trim() + "%");
        LOG.debug("Found {} that have {}", countries.size(), name);
        for (final Country c : countries) {
            try {
                final JsonNode nameJ = mapper.readTree(c.getNameL());
                final Iterator<JsonNode> it = nameJ.elements();
                while (it.hasNext()) {
                    final JsonNode el = it.next();
                    if (name.equalsIgnoreCase(el.textValue())) {
                        LOG.debug("Found match for {} in: {}", name, c.getName());
                        return c;
                    }
                }
            } catch (final IOException e) {
                LOG.error(e.getMessage(), e);
            }
        }
        return null;
    }

    /**
     * Get current country based on ISO3 code. Follow replacedBy where possible.
     *
     * @param code3
     * @return
     */
    @Override
    public Country getCurrentCountry(String code3) {
        if (code3 == null) {
            return null;
        }

        Country country = getCountry(code3);

        if (country != null && country.getReplacedBy() != null) {
            // Loop detection
            final Set<Long> seenCountryId = new HashSet<Long>();
            while (!seenCountryId.contains(country.getId()) && country.getReplacedBy() != null) {
                LOG.info("Country {} replaced by {}", country.getCode3(), country.getReplacedBy());

                // Put countryId to seen list
                seenCountryId.add(country.getId());

                // Update reference
                country = country.getReplacedBy();
            }
        }

        return country;
    }

    @Override
    public Country getCountryByRefnameId(long refnameId) {
        return countryRepository.findByRefnameId(refnameId);
    }

    @Override
    @Transactional(readOnly = false)
    @CacheEvict(value = "statistics", allEntries = true)
    public void updateCountryData() throws IOException {
        // update current countries
        updateGeoNamesCountries();

        // update from Davros, it has info on inactive country codes
        updateDavrosCountries();

        // update custom ISO3316-3 codes
        updateCustomCountryCodes();

        LOG.info("Country data up to date");
    }

    private void updateCustomCountryCodes() throws IOException {
        if (customISO3166Source == null) {
            LOG.warn("Custom ISO3166 reader not available");
            return;
        }

        final List<Custom3166Entry> countries = customISO3166Source.fetchCountryData();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Got {} custom ISO3166 codes from source.", countries.size());
        }

        // check against repository
        for (final Custom3166Entry countryInfo : countries) {
            final Country country = countryRepository.findByCode3(countryInfo.getCode3());

            if (country == null) {
                LOG.info("ISO3166 code {} is not registered: {}", countryInfo.getCode3(), countryInfo);

                final Country newCountry = new Country();
                newCountry.setCode3(countryInfo.getCode3());
                newCountry.setName(countryInfo.getName());
                newCountry.setWikiLink(countryInfo.getUrl());
                newCountry.setCurrent(true);
                countryRepository.save(newCountry);

                LOG.info("Added ISO3166 code {}", newCountry);
            } else {
                LOG.debug("Updating existing custom ISO3166 code {}", country.getCode3());
                
                country.setCurrent(true);
                country.setName(countryInfo.getName());
                country.setWikiLink(countryInfo.getUrl());
                countryRepository.save(country);
            }
        }
    }

    private void updateDavrosCountries() throws IOException {
        if (davrosCountrySource == null) {
            LOG.warn("davros.org country source not available");
            return;
        }

        final List<CountryInfo> countries = davrosCountrySource.fetchCountryData();

        if (LOG.isDebugEnabled()) {
            LOG.debug("Got {} countries from remote source.", countries.size());
        }

        // check against repository
        for (final CountryInfo countryInfo : countries) {

            // Country country =
            // countryRepository.findByName(countryInfo.getName());
            final Country country = countryRepository.findByCode3(countryInfo.getIso3());

            if (country == null) {
                LOG.info("Country {} is not registered: {}", countryInfo.getIso3(), countryInfo);

                if (countryInfo.isActive()) {
                    LOG.warn("Country is marked as active. Should not be.");
                }

                final Country newCountry = new Country();
                newCountry.setCode2(countryInfo.getIso());
                newCountry.setCode3(countryInfo.getIso3());
                newCountry.setCodeNum(countryInfo.getIsoNum());
                newCountry.setCurrent(countryInfo.isActive());
                newCountry.setName(countryInfo.getName());
                countryRepository.save(newCountry);

                LOG.info("Added country {}", newCountry);

            } else {
                LOG.debug("Exists {}", country);

                // if iso2 is blank
                if (StringUtils.isBlank(country.getCode2()) && StringUtils.isNotBlank(countryInfo.getIso())) {
                    LOG.info("Updating country iso2 code");
                    country.setCode2(countryInfo.getIso());
                    countryRepository.save(country);
                }

                // if iso-numeric is blank
                if (StringUtils.isBlank(country.getCodeNum()) && StringUtils.isNotBlank(countryInfo.getIsoNum())) {
                    LOG.info("Updating country iso-numeric code");
                    country.setCodeNum(countryInfo.getIsoNum());
                    countryRepository.save(country);
                }
                /*
                 * // if all fields match if
                 * (country.getCode2().equals(countryInfo.getIso()) &&
                 * country.getCodeNum() != null &&
                 * country.getCodeNum().equals(countryInfo.getIsoNum())) { if
                 * (country.isCurrent() != countryInfo.isActive()) { LOG.warn(
                 * "Country " + country + " is listed as active=" +
                 * countryInfo.isActive() + " on remote site");
                 * country.setCurrent(countryInfo.isActive());
                 * countryRepository.save(country); } }
                 */
            }
        }
    }

    private void updateGeoNamesCountries() throws IOException {
        if (geoNamesCountrySource == null) {
            LOG.warn("geonames.org country source not available");
            return;
        }

        final List<CountryInfo> countries = geoNamesCountrySource.fetchCountryData();

        if (LOG.isDebugEnabled()) {
            LOG.debug("Got {} countries from remote source.", countries.size());
        }

        // deactivate all
        countryRepository.deactivateAll();

        // check against repository
        for (final CountryInfo countryInfo : countries) {
            final Country country = countryRepository.findByCode3(countryInfo.getIso3());

            if (country == null) {
                LOG.info("Country {} is not registered", countryInfo);

                final Country newCountry = new Country();
                newCountry.setCode2(countryInfo.getIso());
                newCountry.setCode3(countryInfo.getIso3());
                newCountry.setCodeNum(countryInfo.getIsoNum());
                newCountry.setCurrent(countryInfo.isActive());
                newCountry.setName(countryInfo.getName());
                newCountry.setRefnameId(countryInfo.getRefnameId());
                countryRepository.save(newCountry);

                LOG.info("Added country {}", newCountry);

            } else {
                LOG.debug("Exists {}", country);
                country.setCurrent(true);
                country.setCode2(countryInfo.getIso());
                country.setCodeNum(countryInfo.getIsoNum());
                country.setName(countryInfo.getName());
                country.setRefnameId(countryInfo.getRefnameId());
                countryRepository.save(country);

                /*
                 * // update refname id if (country.getRefnameId() == null &&
                 * countryInfo.getRefnameId() != null) {
                 * country.setRefnameId(countryInfo.getRefnameId());
                 * countryRepository.save(country); }
                 *
                 * // if country name is not the same if
                 * (StringUtils.isNotBlank(countryInfo.getName()) &&
                 * !countryInfo.getName().equals(country.getName())) {
                 *
                 * LOG.info("Updating country name from: " + country.getName() +
                 * " to: " + countryInfo.getName());
                 * country.setName(countryInfo.getName());
                 * countryRepository.save(country); }
                 */
            }
        }
    }

    @Override
    @PreAuthorize("hasRole('ADMINISTRATOR')")
    @Transactional(readOnly = false)
    public void updateBlurb(Country country, String blurb, Locale locale) throws CRMException {
        // TODO Should we provide summary?
        contentService.updateArticle(country, ContentService.ENTITY_BLURB_SLUG, null, null, blurb, locale);
    }

    @Override
    public List<Long> listCountryRefnameIds() {
        return countryRepository.listRefnameIds();
    }

    @Override
    // TODO Where do we autorize access?
    // @PreAuthorize("hasRole('ADMINISTRATOR')")
    @Transactional(readOnly = false)
    public void updateCountryNames(String isoCode3, String jsonTranslations) {
        Country country = countryRepository.findByCode3(isoCode3);
        country.setNameL(jsonTranslations);
        country = countryRepository.save(country);
        LOG.info("Updated translations of {} i18n={}", country, country.getNameL());
    }

    @Override
    @Transactional(readOnly = false)
    public Country updateCountryWiki(String isoCode3, String wiki) {
        Country country = countryRepository.findByCode3(isoCode3);
        LOG.info("Loaded {} i18n={}", country, country.getNameL());
        country.setWikiLink(wiki);
        country = countryRepository.save(country);
        LOG.info("Updated wiki link of {} i18n={}", country, country.getNameL());
        return country;
    }

    @Override
    public ArrayNode toJson(List<FaoInstitute> members) {
        // Generate JSON
        final ArrayNode jsonArray = mapper.createArrayNode();
        for (final FaoInstitute inst : members) {
            if (inst.getLatitude() != null) {
                final ObjectNode instNode = mapper.createObjectNode();
                instNode.put("lat", inst.getLatitude());
                instNode.put("lng", inst.getLongitude());
                instNode.put("elevation", inst.getElevation());
                instNode.put("title", inst.getFullName());
                instNode.put("code", inst.getCode());
                jsonArray.add(instNode);
            }
        }
        return jsonArray;
    }

    @Override
    public ITPGRFAStatus getITPGRFAStatus(Country country) {
        return itpgrfaRepository.findByCountry(country);
    }

    @Override
    @Transactional(readOnly = false)
    public ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy) {
        if (country == null) {
            LOG.warn("Country is null, not updating ITPGRFA");
            return null;
        }

        ITPGRFAStatus itpgrfaStatus = itpgrfaRepository.findByCountry(country);
        if (itpgrfaStatus == null) {
            LOG.info("New ITPGRFA entry for {}", country.getName());
            itpgrfaStatus = new ITPGRFAStatus();
            itpgrfaStatus.setCountry(country);
        } else {
            LOG.info("Updating ITPGRFA entry for {}", country.getName());
        }
        itpgrfaStatus.setContractingParty(contractingParty);
        itpgrfaStatus.setMembership(membership);
        itpgrfaStatus.setMembershipBy(membershipBy);

        return itpgrfaRepository.save(itpgrfaStatus);
    }

    @Override
    public String filteredKml(String jsonFilter) {
        return null;
    }

    @Override
    public List<Country> autocomplete(String term) {
        CountryFilter filter = new CountryFilter();
        term = StringUtils.strip(term);
        if (term.length() == 0) {
            return List.of();
        }
        filter._text(term + " | " + term + "*");
        List<Country> autocompleteCountries = new ArrayList<>();
        try {
            autocompleteCountries = elasticsearchService.find(Country.class, filter);
        } catch (SearchException e) {
            LOG.error("Error in searching countries with term: {}", term);
        }
        return autocompleteCountries;
    }

    @Override
    public List<VocabularyTerm> autoCompleteTerm(String ac) {
        return autocomplete(ac).stream().map((country) -> toVocabularyTerm(country, country.getCode3())).collect(Collectors.toList());
    }

    @Override
    public CountryDetails getDetails(String iso3code) {
        Country country = getCountry(iso3code);
        if (country == null) {
            throw new NotFoundElement("Cannot find country by ISO code " + iso3code);
        }

        AccessionFilter byCountry = new AccessionFilter();
        byCountry.countryOfOrigin(new CountryFilter())
            .countryOfOrigin.code3 = Sets.newHashSet(country.getCode3());
        Long accessionCount = null;
        try {
            accessionCount = accessionService.countAccessions(byCountry);
        } catch (SearchException e) {
            LOG.warn("Error occurred during search", e);
        }
        long countByLocation = genesysService.countByLocation(country);

        Map<String, ElasticsearchService.TermResult> overview = getOverviewData(byCountry);
        List<FaoInstitute> genesysInstitutes = instituteService.listByCountryActive(country);
        List<FaoInstitute> faoInstitutes = instituteService.listByCountry(country);

        return CountryDetails.from(country, getITPGRFAStatus(country), accessionCount, countByLocation, faoInstitutes, genesysInstitutes, overview);
    }

    private Map<String, ElasticsearchService.TermResult> getOverviewData(AccessionFilter byCountryFilter) {
        String[] terms = new String[] {"taxonomy.genus", "taxonomy.genusSpecies", "institute.code",
            "institute.country.code3", "mlsStatus", "available"};

        if (elasticsearchService == null)
            return Map.of();

        try {
            return elasticsearchService.termStatisticsAuto(Accession.class, byCountryFilter, 10, terms);
        } catch (SearchException e) {
            LOG.error("Error occurred during search", e);
            return null;
        }
    }

    @Override
    public String getBoundingBox(final Set<String> iso3Codes) {
        List<String> countryIso3List = new ArrayList<>(iso3Codes);

        Set<Double> minLatitudeSet = new TreeSet<>(Double::compareTo);
        Set<Double> minLongitudeSet = new TreeSet<>(Double::compareTo);
        Set<Double> maxLongitudeSet = new TreeSet<>(Comparator.reverseOrder());
        Set<Double> maxLatitudeSet = new TreeSet<>(Comparator.reverseOrder());

        for (String iso3: countryIso3List) {
            Country country = getCountry(iso3);
            if (country != null) {
                if (country.getMinLatitude() != null) minLatitudeSet.add(country.getMinLatitude());
                if (country.getMinLongitude() != null) minLongitudeSet.add(country.getMinLongitude());
                if (country.getMaxLatitude() != null) maxLatitudeSet.add(country.getMaxLatitude());
                if (country.getMaxLongitude() != null) maxLongitudeSet.add(country.getMaxLongitude());
            }
        }

        final ObjectNode geoJson = mapper.createObjectNode();
        geoJson.put("north", maxLatitudeSet.stream().findFirst().orElse(0.0));
        geoJson.put("south", minLatitudeSet.stream().findFirst().orElse(0.0));
        geoJson.put("east", maxLongitudeSet.stream().findFirst().orElse(0.0));
        geoJson.put("west", minLongitudeSet.stream().findFirst().orElse(0.0));
        return geoJson.toString();
    }

    private VocabularyTerm toVocabularyTerm(Country country, String code) {

        if (country == null) {
            throw new NotFoundElement("No such country term");
        }

        VocabularyTerm countryTerm = new VocabularyTerm();

        countryTerm.setCode(code);
        countryTerm.setTitle(country.getName());

        return countryTerm;
    }

    @Override
    public Page<VocabularyTerm> list3166Alpha2Terms(Pageable page) {
        Page<Country> res = countryRepository.findAll(QCountry.country.code2.isNotNull(), page);
        return res.map(c -> toVocabularyTerm(c, c.getCode2()));
    }

    @Override
    public Page<VocabularyTerm> list3166Alpha3Terms(Pageable page) {
        Page<Country> res = countryRepository.findAll(QCountry.country.code3.isNotNull(), page);
        return res.map(c -> toVocabularyTerm(c, c.getCode3()));
    }

    @Override
    public Map<String, String> decode3166Alpha3Terms(Set<String> codes, Locale locale) {
        Predicate whereClause = codes != null ? QCountry.country.code3.in(codes) : null;

        List<Tuple> query = jpaQueryFactory.select(QCountry.country.code3, QCountry.country.nameL, QCountry.country.name)
                .from(QCountry.country)
                .where(whereClause)
                .fetch();
        return query.stream().collect(
                Collectors.toMap(
                        tuple -> tuple.get(QCountry.country.code3),
                        tuple -> decodeNameLocal(locale, tuple.get(QCountry.country.nameL), tuple.get(QCountry.country.name))
                )
        );
    }

    @Override
    public Page<VocabularyTerm> list3166NumericTerms(Pageable page) {
        Page<Country> res = countryRepository.findAll(QCountry.country.codeNum.isNotNull(), page);
        return res.map(c -> toVocabularyTerm(c, c.getCodeNum() != null ? c.getCodeNum().toString() : null));

    }

    @Override
    public VocabularyTerm get3166Alpha2Term(String code) {
        Country c = countryRepository.findByCode2(code);
        return c == null ? null : toVocabularyTerm(c, c.getCode2());
    }

    @Override
    public VocabularyTerm get3166Alpha3Term(String code) {
        Country c = countryRepository.findByCode3(code);
        return c == null ? null : toVocabularyTerm(c, c.getCode3());
    }

    @Override
    public VocabularyTerm get3166NumericTerm(String code) {
        Country c = countryRepository.findByCodeNum(code);
        return c == null ? null : toVocabularyTerm(c, c.getCodeNum().toString());
    }

    private String decodeNameLocal(Locale locale, String nameL, String name){
        if (nameL == null)
            return name;

        try {
            JsonNode nameJ = mapper.readTree(nameL);
            if (nameJ.has(locale.toLanguageTag())) {
                return nameJ.get(locale.toLanguageTag()).textValue();
            } else if (nameJ.has(locale.getLanguage())) {
                return nameJ.get(locale.getLanguage()).textValue();
            }
            return name;
        } catch (IOException e) {
            LOG.warn("Error while decoding country code: {}", e.getMessage());
            return  name;
        }
    }

}