GeoServiceImpl.java

/*
 * Copyright 2018 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.service.impl;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;

import org.apache.commons.lang3.StringUtils;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.Country;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.model.impl.ITPGRFAStatus;
import org.genesys.server.model.impl.QCountry;
import org.genesys.server.model.vocab.VocabularyTerm;
import org.genesys.server.persistence.CountryRepository;
import org.genesys.server.persistence.ITPGRFAStatusRepository;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.CRMException;
import org.genesys.server.service.ContentService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.GeoService;
import org.genesys.server.service.InstituteService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.service.filter.CountryFilter;
import org.genesys.server.service.worker.CountryInfo;
import org.genesys.server.service.worker.CustomISO3166Source;
import org.genesys.server.service.worker.CustomISO3166Source.Custom3166Entry;
import org.genesys.server.service.worker.DavrosCountrySource;
import org.genesys.server.service.worker.GeoNamesCountrySource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Sets;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.impl.JPAQueryFactory;

@Service
@Transactional(readOnly = true)
public class GeoServiceImpl implements GeoService {
	public static final Logger LOG = LoggerFactory.getLogger(GeoServiceImpl.class);

	@Autowired
	private JPAQueryFactory jpaQueryFactory;

	@Autowired
	CountryRepository countryRepository;

	@Autowired
	ContentService contentService;

	@Autowired
	ITPGRFAStatusRepository itpgrfaRepository;

	@Autowired(required = false)
	private GeoNamesCountrySource geoNamesCountrySource;

	@Autowired(required = false)
	private DavrosCountrySource davrosCountrySource;

	@Autowired(required = false)
	private CustomISO3166Source customISO3166Source;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired
	private InstituteService instituteService;

	@Autowired
	private AccessionService accessionService;

	@Autowired
	private GenesysService genesysService;

	private final ObjectMapper mapper = new ObjectMapper();

	@Override
	public List<Country> listAll() {
		return countryRepository.findAll(Sort.by("name", "current"));
	}

	@Override
	public List<Country> listAll(final Locale locale) {
		final List<Country> all = listAll();
		Collections.sort(all, new Comparator<Country>() {
			@Override
			public int compare(Country o1, Country o2) {
				return o1.getName(locale).compareTo(o2.getName(locale));
			}
		});
		return all;
	}

	@Override
	public List<Country> listActive(final Locale locale) {
		final List<Country> countries = countryRepository.findByCurrent(true);
		Country.sort(countries, locale);
		return countries;
	}

	@Override
	public long countActive() {
		return countryRepository.countByCurrent(true);
	}

	@Override
	public List<Country> listITPGRFA(Locale locale) {
		final List<Country> countries = countryRepository.findITPGRFA();
		Country.sort(countries, locale);
		return countries;
	}

	@Override
	public Country getCountry(String isoCode) {
		Country country = null;
		if (isoCode.length() == 3) {
			country = countryRepository.findByCode3(isoCode);
		} else if (isoCode.length() == 2) {
			// RO = ROU and ROM
			country = countryRepository.findByCode2AndCurrent(isoCode, true);
			if (country == null) {
				country = countryRepository.findByCode2AndCurrent(isoCode, false);
			}
		}

		return country;
	}

	@Override
	public Country findCountry(String countryString) {
		Country country = getCountry(countryString);

		// Let's try the name
		if (country == null) {
			country = countryRepository.findByName(countryString);
		}

		// Let's try translations
		if (country == null) {
			country = findCountryByName(countryString);
		}
		
		// Let's sanitize input
		if (country == null) {
			country = findCountryByName(countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
			if (country != null) {
				LOG.warn("Found country {} for {}", country.getName(), countryString);
			} else {
				LOG.warn("No country for {} - queried by {}", countryString, countryString.replaceAll("\\s*\\(.[^\\)]+\\)", ""));
			}
		}

		return country;
	}

	/**
	 * Check if we have a country that has
	 *
	 * @param name in i18n
	 * @return
	 */
	private Country findCountryByName(String name) {
		final List<Country> countries = countryRepository.findWithI18N("%" + name.trim() + "%");
		LOG.debug("Found {} that have {}", countries.size(), name);
		for (final Country c : countries) {
			try {
				final JsonNode nameJ = mapper.readTree(c.getNameL());
				final Iterator<JsonNode> it = nameJ.elements();
				while (it.hasNext()) {
					final JsonNode el = it.next();
					if (name.equalsIgnoreCase(el.textValue())) {
						LOG.debug("Found match for {} in: {}", name, c.getName());
						return c;
					}
				}
			} catch (final IOException e) {
				LOG.error(e.getMessage(), e);
			}
		}
		return null;
	}

	/**
	 * Get current country based on ISO3 code. Follow replacedBy where possible.
	 *
	 * @param code3
	 * @return
	 */
	@Override
	public Country getCurrentCountry(String code3) {
		if (code3 == null) {
			return null;
		}

		Country country = getCountry(code3);

		if (country != null && country.getReplacedBy() != null) {
			// Loop detection
			final Set<Long> seenCountryId = new HashSet<Long>();
			while (!seenCountryId.contains(country.getId()) && country.getReplacedBy() != null) {
				LOG.info("Country {} replaced by {}", country.getCode3(), country.getReplacedBy());

				// Put countryId to seen list
				seenCountryId.add(country.getId());

				// Update reference
				country = country.getReplacedBy();
			}
		}

		return country;
	}

	@Override
	public Country getCountryByRefnameId(long refnameId) {
		return countryRepository.findByRefnameId(refnameId);
	}

	@Override
	@Transactional(readOnly = false)
	@CacheEvict(value = "statistics", allEntries = true)
	public void updateCountryData() throws IOException {
		// update current countries
		updateGeoNamesCountries();

		// update from Davros, it has info on inactive country codes
		updateDavrosCountries();

		// update custom ISO3316-3 codes
		updateCustomCountryCodes();

		LOG.info("Country data up to date");
	}

	private void updateCustomCountryCodes() throws IOException {
		if (customISO3166Source == null) {
			LOG.warn("Custom ISO3166 reader not available");
			return;
		}

		final List<Custom3166Entry> countries = customISO3166Source.fetchCountryData();
		if (LOG.isDebugEnabled()) {
			LOG.debug("Got {} custom ISO3166 codes from source.", countries.size());
		}

		// check against repository
		for (final Custom3166Entry countryInfo : countries) {
			final Country country = countryRepository.findByCode3(countryInfo.getCode3());

			if (country == null) {
				LOG.info("ISO3166 code {} is not registered: {}", countryInfo.getCode3(), countryInfo);

				final Country newCountry = new Country();
				newCountry.setCode3(countryInfo.getCode3());
				newCountry.setName(countryInfo.getName());
				newCountry.setWikiLink(countryInfo.getUrl());
				newCountry.setCurrent(true);
				countryRepository.save(newCountry);

				LOG.info("Added ISO3166 code {}", newCountry);
			} else {
				LOG.debug("Updating existing custom ISO3166 code {}", country.getCode3());
				
				country.setCurrent(true);
				country.setName(countryInfo.getName());
				country.setWikiLink(countryInfo.getUrl());
				countryRepository.save(country);
			}
		}
	}

	private void updateDavrosCountries() throws IOException {
		if (davrosCountrySource == null) {
			LOG.warn("davros.org country source not available");
			return;
		}

		final List<CountryInfo> countries = davrosCountrySource.fetchCountryData();

		if (LOG.isDebugEnabled()) {
			LOG.debug("Got {} countries from remote source.", countries.size());
		}

		// check against repository
		for (final CountryInfo countryInfo : countries) {

			// Country country =
			// countryRepository.findByName(countryInfo.getName());
			final Country country = countryRepository.findByCode3(countryInfo.getIso3());

			if (country == null) {
				LOG.info("Country {} is not registered: {}", countryInfo.getIso3(), countryInfo);

				if (countryInfo.isActive()) {
					LOG.warn("Country is marked as active. Should not be.");
				}

				final Country newCountry = new Country();
				newCountry.setCode2(countryInfo.getIso());
				newCountry.setCode3(countryInfo.getIso3());
				newCountry.setCodeNum(countryInfo.getIsoNum());
				newCountry.setCurrent(countryInfo.isActive());
				newCountry.setName(countryInfo.getName());
				countryRepository.save(newCountry);

				LOG.info("Added country {}", newCountry);

			} else {
				LOG.debug("Exists {}", country);

				// if iso2 is blank
				if (StringUtils.isBlank(country.getCode2()) && StringUtils.isNotBlank(countryInfo.getIso())) {
					LOG.info("Updating country iso2 code");
					country.setCode2(countryInfo.getIso());
					countryRepository.save(country);
				}

				// if iso-numeric is blank
				if (StringUtils.isBlank(country.getCodeNum()) && StringUtils.isNotBlank(countryInfo.getIsoNum())) {
					LOG.info("Updating country iso-numeric code");
					country.setCodeNum(countryInfo.getIsoNum());
					countryRepository.save(country);
				}
				/*
				 * // if all fields match if
				 * (country.getCode2().equals(countryInfo.getIso()) &&
				 * country.getCodeNum() != null &&
				 * country.getCodeNum().equals(countryInfo.getIsoNum())) { if
				 * (country.isCurrent() != countryInfo.isActive()) { LOG.warn(
				 * "Country " + country + " is listed as active=" +
				 * countryInfo.isActive() + " on remote site");
				 * country.setCurrent(countryInfo.isActive());
				 * countryRepository.save(country); } }
				 */
			}
		}
	}

	private void updateGeoNamesCountries() throws IOException {
		if (geoNamesCountrySource == null) {
			LOG.warn("geonames.org country source not available");
			return;
		}

		final List<CountryInfo> countries = geoNamesCountrySource.fetchCountryData();

		if (LOG.isDebugEnabled()) {
			LOG.debug("Got {} countries from remote source.", countries.size());
		}

		// deactivate all
		countryRepository.deactivateAll();

		// check against repository
		for (final CountryInfo countryInfo : countries) {
			final Country country = countryRepository.findByCode3(countryInfo.getIso3());

			if (country == null) {
				LOG.info("Country {} is not registered", countryInfo);

				final Country newCountry = new Country();
				newCountry.setCode2(countryInfo.getIso());
				newCountry.setCode3(countryInfo.getIso3());
				newCountry.setCodeNum(countryInfo.getIsoNum());
				newCountry.setCurrent(countryInfo.isActive());
				newCountry.setName(countryInfo.getName());
				newCountry.setRefnameId(countryInfo.getRefnameId());
				countryRepository.save(newCountry);

				LOG.info("Added country {}", newCountry);

			} else {
				LOG.debug("Exists {}", country);
				country.setCurrent(true);
				country.setCode2(countryInfo.getIso());
				country.setCodeNum(countryInfo.getIsoNum());
				country.setName(countryInfo.getName());
				country.setRefnameId(countryInfo.getRefnameId());
				countryRepository.save(country);

				/*
				 * // update refname id if (country.getRefnameId() == null &&
				 * countryInfo.getRefnameId() != null) {
				 * country.setRefnameId(countryInfo.getRefnameId());
				 * countryRepository.save(country); }
				 *
				 * // if country name is not the same if
				 * (StringUtils.isNotBlank(countryInfo.getName()) &&
				 * !countryInfo.getName().equals(country.getName())) {
				 *
				 * LOG.info("Updating country name from: " + country.getName() +
				 * " to: " + countryInfo.getName());
				 * country.setName(countryInfo.getName());
				 * countryRepository.save(country); }
				 */
			}
		}
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR')")
	@Transactional(readOnly = false)
	public void updateBlurb(Country country, String blurb, Locale locale) throws CRMException {
		// TODO Should we provide summary?
		contentService.updateArticle(country, ContentService.ENTITY_BLURB_SLUG, null, null, blurb, locale);
	}

	@Override
	public List<Long> listCountryRefnameIds() {
		return countryRepository.listRefnameIds();
	}

	@Override
	// TODO Where do we autorize access?
	// @PreAuthorize("hasRole('ADMINISTRATOR')")
	@Transactional(readOnly = false)
	public void updateCountryNames(String isoCode3, String jsonTranslations) {
		Country country = countryRepository.findByCode3(isoCode3);
		country.setNameL(jsonTranslations);
		country = countryRepository.save(country);
		LOG.info("Updated translations of {} i18n={}", country, country.getNameL());
	}

	@Override
	@Transactional(readOnly = false)
	public Country updateCountryWiki(String isoCode3, String wiki) {
		Country country = countryRepository.findByCode3(isoCode3);
		LOG.info("Loaded {} i18n={}", country, country.getNameL());
		country.setWikiLink(wiki);
		country = countryRepository.save(country);
		LOG.info("Updated wiki link of {} i18n={}", country, country.getNameL());
		return country;
	}

	@Override
	public ArrayNode toJson(List<FaoInstitute> members) {
		// Generate JSON
		final ArrayNode jsonArray = mapper.createArrayNode();
		for (final FaoInstitute inst : members) {
			if (inst.getLatitude() != null) {
				final ObjectNode instNode = mapper.createObjectNode();
				instNode.put("lat", inst.getLatitude());
				instNode.put("lng", inst.getLongitude());
				instNode.put("elevation", inst.getElevation());
				instNode.put("title", inst.getFullName());
				instNode.put("code", inst.getCode());
				jsonArray.add(instNode);
			}
		}
		return jsonArray;
	}

	@Override
	public ITPGRFAStatus getITPGRFAStatus(Country country) {
		return itpgrfaRepository.findByCountry(country);
	}

	@Override
	@Transactional(readOnly = false)
	public ITPGRFAStatus updateITPGRFA(Country country, String contractingParty, String membership, String membershipBy) {
		if (country == null) {
			LOG.warn("Country is null, not updating ITPGRFA");
			return null;
		}

		ITPGRFAStatus itpgrfaStatus = itpgrfaRepository.findByCountry(country);
		if (itpgrfaStatus == null) {
			LOG.info("New ITPGRFA entry for {}", country.getName());
			itpgrfaStatus = new ITPGRFAStatus();
			itpgrfaStatus.setCountry(country);
		} else {
			LOG.info("Updating ITPGRFA entry for {}", country.getName());
		}
		itpgrfaStatus.setContractingParty(contractingParty);
		itpgrfaStatus.setMembership(membership);
		itpgrfaStatus.setMembershipBy(membershipBy);

		return itpgrfaRepository.save(itpgrfaStatus);
	}

	@Override
	public String filteredKml(String jsonFilter) {
		return null;
	}

	@Override
	public List<Country> autocomplete(String term) {
		CountryFilter filter = new CountryFilter();
		term = StringUtils.strip(term);
		if (term.length() == 0) {
			return List.of();
		}
		filter._text(term + " | " + term + "*");
		List<Country> autocompleteCountries = new ArrayList<>();
		try {
			autocompleteCountries = elasticsearchService.find(Country.class, filter);
		} catch (SearchException e) {
			LOG.error("Error in searching countries with term: {}", term);
		}
		return autocompleteCountries;
	}

	@Override
	public List<VocabularyTerm> autoCompleteTerm(String ac) {
		return autocomplete(ac).stream().map((country) -> toVocabularyTerm(country, country.getCode3())).collect(Collectors.toList());
	}

	@Override
	public CountryDetails getDetails(String iso3code) {
		Country country = getCountry(iso3code);
		if (country == null) {
			throw new NotFoundElement("Cannot find country by ISO code " + iso3code);
		}

		AccessionFilter byCountry = new AccessionFilter();
		byCountry.countryOfOrigin(new CountryFilter())
			.countryOfOrigin.code3 = Sets.newHashSet(country.getCode3());
		Long accessionCount = null;
		try {
			accessionCount = accessionService.countAccessions(byCountry);
		} catch (SearchException e) {
			LOG.warn("Error occurred during search", e);
		}
		long countByLocation = genesysService.countByLocation(country);

		Map<String, ElasticsearchService.TermResult> overview = getOverviewData(byCountry);
		List<FaoInstitute> genesysInstitutes = instituteService.listByCountryActive(country);
		List<FaoInstitute> faoInstitutes = instituteService.listByCountry(country);

		return CountryDetails.from(country, getITPGRFAStatus(country), accessionCount, countByLocation, faoInstitutes, genesysInstitutes, overview);
	}

	private Map<String, ElasticsearchService.TermResult> getOverviewData(AccessionFilter byCountryFilter) {
		String[] terms = new String[] {"taxonomy.genus", "taxonomy.genusSpecies", "institute.code",
			"institute.country.code3", "mlsStatus", "available"};

		if (elasticsearchService == null)
			return Map.of();

		try {
			return elasticsearchService.termStatisticsAuto(Accession.class, byCountryFilter, 10, terms);
		} catch (SearchException e) {
			LOG.error("Error occurred during search", e);
			return null;
		}
	}

	@Override
	public String getBoundingBox(final Set<String> iso3Codes) {
		List<String> countryIso3List = new ArrayList<>(iso3Codes);

		Set<Double> minLatitudeSet = new TreeSet<>(Double::compareTo);
		Set<Double> minLongitudeSet = new TreeSet<>(Double::compareTo);
		Set<Double> maxLongitudeSet = new TreeSet<>(Comparator.reverseOrder());
		Set<Double> maxLatitudeSet = new TreeSet<>(Comparator.reverseOrder());

		for (String iso3: countryIso3List) {
			Country country = getCountry(iso3);
			if (country != null) {
				if (country.getMinLatitude() != null) minLatitudeSet.add(country.getMinLatitude());
				if (country.getMinLongitude() != null) minLongitudeSet.add(country.getMinLongitude());
				if (country.getMaxLatitude() != null) maxLatitudeSet.add(country.getMaxLatitude());
				if (country.getMaxLongitude() != null) maxLongitudeSet.add(country.getMaxLongitude());
			}
		}

		final ObjectNode geoJson = mapper.createObjectNode();
		geoJson.put("north", maxLatitudeSet.stream().findFirst().orElse(0.0));
		geoJson.put("south", minLatitudeSet.stream().findFirst().orElse(0.0));
		geoJson.put("east", maxLongitudeSet.stream().findFirst().orElse(0.0));
		geoJson.put("west", minLongitudeSet.stream().findFirst().orElse(0.0));
		return geoJson.toString();
	}

	private VocabularyTerm toVocabularyTerm(Country country, String code) {

		if (country == null) {
			throw new NotFoundElement("No such country term");
		}

		VocabularyTerm countryTerm = new VocabularyTerm();

		countryTerm.setCode(code);
		countryTerm.setTitle(country.getName());

		return countryTerm;
	}

	@Override
	public Page<VocabularyTerm> list3166Alpha2Terms(Pageable page) {
		Page<Country> res = countryRepository.findAll(QCountry.country.code2.isNotNull(), page);
		return res.map(c -> toVocabularyTerm(c, c.getCode2()));
	}

	@Override
	public Page<VocabularyTerm> list3166Alpha3Terms(Pageable page) {
		Page<Country> res = countryRepository.findAll(QCountry.country.code3.isNotNull(), page);
		return res.map(c -> toVocabularyTerm(c, c.getCode3()));
	}

	@Override
	public Map<String, String> decode3166Alpha3Terms(Set<String> codes, Locale locale) {
		Predicate whereClause = codes != null ? QCountry.country.code3.in(codes) : null;

		List<Tuple> query = jpaQueryFactory.select(QCountry.country.code3, QCountry.country.nameL, QCountry.country.name)
				.from(QCountry.country)
				.where(whereClause)
				.fetch();
		return query.stream().collect(
				Collectors.toMap(
						tuple -> tuple.get(QCountry.country.code3),
						tuple -> decodeNameLocal(locale, tuple.get(QCountry.country.nameL), tuple.get(QCountry.country.name))
				)
		);
    }

    @Override
	public Page<VocabularyTerm> list3166NumericTerms(Pageable page) {
		Page<Country> res = countryRepository.findAll(QCountry.country.codeNum.isNotNull(), page);
		return res.map(c -> toVocabularyTerm(c, c.getCodeNum() != null ? c.getCodeNum().toString() : null));

	}

	@Override
	public VocabularyTerm get3166Alpha2Term(String code) {
		Country c = countryRepository.findByCode2(code);
		return c == null ? null : toVocabularyTerm(c, c.getCode2());
	}

	@Override
	public VocabularyTerm get3166Alpha3Term(String code) {
		Country c = countryRepository.findByCode3(code);
		return c == null ? null : toVocabularyTerm(c, c.getCode3());
	}

	@Override
	public VocabularyTerm get3166NumericTerm(String code) {
		Country c = countryRepository.findByCodeNum(code);
		return c == null ? null : toVocabularyTerm(c, c.getCodeNum().toString());
	}

	private String decodeNameLocal(Locale locale, String nameL, String name){
		if (nameL == null)
			return name;

		try {
			JsonNode nameJ = mapper.readTree(nameL);
			if (nameJ.has(locale.toLanguageTag())) {
				return nameJ.get(locale.toLanguageTag()).textValue();
			} else if (nameJ.has(locale.getLanguage())) {
				return nameJ.get(locale.getLanguage()).textValue();
			}
			return name;
		} catch (IOException e) {
			LOG.warn("Error while decoding country code: {}", e.getMessage());
			return  name;
		}
	}

}