AccessionController.java

/*
 * Copyright 2024 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.api.v2.impl;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
import com.opencsv.CSVWriter;
import io.swagger.annotations.Api;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.serialization.CurrentPermissionsWriter;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.FilteredPage;
import org.genesys.server.api.FilteredSlice;
import org.genesys.server.api.Pagination;
import org.genesys.server.api.ScrollPagination;
import org.genesys.server.api.model.AccessionHeaderJson;
import org.genesys.server.api.v2.facade.AccessionApiService;
import org.genesys.server.api.v2.model.impl.AccessionDTO;
import org.genesys.server.api.v2.model.impl.Taxonomy2Info;
import org.genesys.server.component.aspect.DownloadEndpoint;
import org.genesys.server.exception.ClientDisconnectedException;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.genesys.AccessionId;
import org.genesys.server.model.genesys.QAccession;
import org.genesys.server.model.impl.AccessionIdentifier3;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.AmphibianService;
import org.genesys.server.service.DownloadService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.ShortFilterService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.service.filter.AccessionGeoFilter;
import org.genesys.server.service.impl.AmphibianServiceImpl;
import org.genesys.server.service.worker.AccessionProcessor;
import org.genesys.server.service.worker.ShortFilterProcessor;
import org.genesys.server.service.worker.dupe.AccessionDuplicateFinder;
import org.genesys.server.service.worker.dupe.DuplicateFinder;
import org.genesys.spring.CSVMessageConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springdoc.api.annotations.ParameterObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.HandlerMapping;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedWriter;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

/**
 * Accession API v2
 */
@RestController("accessionApi2")
@PreAuthorize("isAuthenticated()")
@RequestMapping(AccessionController.CONTROLLER_URL)
@Api(tags = { "accession" })
public class AccessionController extends ApiBaseController {

	private static final Logger LOG = LoggerFactory.getLogger(AccessionController.class);

	/** The Constant CONTROLLER_URL. */
	public static final String CONTROLLER_URL = ApiBaseController.APIv2_BASE + "/acn";

	private static final int DOWNLOAD_LIMIT = 300000;

	@Autowired
	private AccessionApiService accessionApiService;

	@Autowired
	protected ShortFilterProcessor shortFilterProcessor;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired
	private DownloadService downloadService;

	@Autowired
	private GenesysService genesysService;

	@Autowired
	private AmphibianService amphibianService;

	@Autowired
	private AccessionProcessor accessionProcessor;

	@Autowired
	private ObjectMapper objectMapper;

	private final Cache<String, AccessionService.AccessionOverview> accessionOverviewCache = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(5, TimeUnit.MINUTES).build();
	private final Cache<String, AccessionService.AccessionMapInfo> accessionMapinfoCache = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(5, TimeUnit.MINUTES).build();

	private final ObjectMapper mapper = new ObjectMapper();

	@Value("${frontend.url}")
	private String frontendUrl;

	@Value("${cdn.servers}")
	private String[] cdnServers;

	public static final Set<String> terms = Sets.newHashSet("institute.code", "institute.country.code3", "cropName", "crop.shortName", "taxonomy.genus", "taxonomy.species",
		"taxonomy.genusSpecies", "taxonomy.grinTaxonomySpecies.name", "taxonomy.currentTaxonomySpecies.name", "countryOfOrigin.code3", "sampStat", "available", "mlsStatus",
		"donorCode", "sgsv", "storage", "duplSite", "breederCode", "aegis", "curationType");

	@Autowired
	private AccessionDuplicateFinder duplicateFinder;

	@GetMapping(value = "/id/{id}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public UUID uuidFromId(@PathVariable("id") final long id) {
		return accessionApiService.uuidFromId(id);
	}

	@PostMapping(value = "/id", produces = { MediaType.APPLICATION_JSON_VALUE })
	public List<UUID> uuidFromIds(@RequestBody List<Long> ids) {
		return accessionApiService.uuidsFromIds(ids);
	}


	@GetMapping(value = "/acce-number/{acceNumber}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public UUID uuidFromAcceNumber(@RequestParam(value = "instCode", required = false) String instCode , @PathVariable("acceNumber") String acceNumber) {
		return accessionApiService.uuidFromAcceNumber(instCode, acceNumber);
	}

	@PostMapping(value = "/acce-number", produces = { MediaType.APPLICATION_JSON_VALUE })
	public List<UUID> uuidsFromAcceNumbers(@RequestParam(value = "instCode", required = false) String instCode, @RequestBody List<String> acceNumbers) {
		return accessionApiService.uuidsFromAcceNumbers(instCode, acceNumbers);
	}

	/**
	 * Gets the accession
	 *
	 * @param uuid the uuid
	 * @return the subset
	 */
	@GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionDTO getByUuid(@PathVariable("uuid") final UUID uuid) {
		return accessionApiService.getByUuid(uuid);
	}

	/**
	 * Gets the accession
	 *
	 * @return the subset
	 */
	@GetMapping(value = "/10.{doi1:[0-9]+}/**", produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionDTO getByDoi(final HttpServletRequest request) {
		final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/").length());
		return accessionApiService.getByDoi(doi);
	}

	/**
	 * List accessions by filterCode or filter
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws IOException
	 */
	@PostMapping(value = "/list", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
	public FilteredPage<AccessionDTO, AccessionFilter> list(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
		@RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
	}


	/**
	 * Query accession fields 
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param select list of {@code Accession} fields to include in the response
	 * @param mcpd Concatenate arrays with ";". Default is <b>{@code false}</b>
	 * @return the page
	 * @throws Exception
	 */
	@PostMapping(value = "/query", produces = { MediaType.APPLICATION_JSON_VALUE })
	public FilteredPage<?, AccessionFilter> query(
		final Pagination page,
		@RequestParam(name = "f", required = false) String filterCode,
		@RequestBody(required = false) AccessionFilter filter,
		@RequestParam List<String> select,
		@RequestParam(required = false, defaultValue = "false") boolean mcpd,
		HttpServletRequest request
	) throws Exception {
		LOG.warn("Using MCPD style? {}", mcpd);
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
//		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionService.query(filterInfo.filter, select, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE), mcpd));
		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.query(filterInfo.filter, select, page.toPageRequest(10000, DEFAULT_PAGE_SIZE), mcpd));
	}

	/**
	 * Query accession fields (for HTTP Form POST)
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param select list of {@code Accession} fields to include in the response
	 * @param mcpd Concatenate arrays with ";". Default is <b>{@code true}</b>
	 * @throws Exception
	 */
	@PostMapping(value = "/query", produces = { CSVMessageConverter.TEXT_CSV_VALUE, CSVMessageConverter.TEXT_TSV_VALUE }, consumes = { MediaType.APPLICATION_FORM_URLENCODED_VALUE, MediaType.MULTIPART_FORM_DATA_VALUE })
	public void queryCsvForm(
		final Pagination page,
		@RequestParam(name = "f", required = false) String filterCode,
		@RequestParam String filter,
		@RequestParam List<String> select,
		@RequestParam(required = false, defaultValue = "true") boolean mcpd,
		HttpServletRequest request,
		HttpServletResponse response
	) throws Exception {
		queryCsv(page, filterCode, objectMapper.readValue(filter, AccessionFilter.class), select, mcpd, request, response);
	}
	/**
	 * Query accession fields. 
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param select list of {@code Accession} fields to include in the response
	 * @param mcpd Concatenate arrays with ";". Default is <b>{@code true}</b>
	 * @throws Exception
	 */
	@PostMapping(value = "/query", produces = { CSVMessageConverter.TEXT_CSV_VALUE, CSVMessageConverter.TEXT_TSV_VALUE })
	public void queryCsv(
		final Pagination page,
		@RequestParam(name = "f", required = false) String filterCode,
		@RequestBody(required = false) AccessionFilter filter,
		@RequestParam List<String> select,
		@RequestParam(required = false, defaultValue = "true") boolean mcpd,
		HttpServletRequest request,
		HttpServletResponse response
	) throws Exception {
		LOG.debug("Using MCPD style? {}", mcpd);
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		response.addHeader("Content-Type", CSVMessageConverter.TEXT_TSV_VALUE);
		String fileName = "query" + (StringUtils.isNotBlank(filterInfo.filterCode) ? "-".concat(filterInfo.filterCode) : "");
		response.addHeader("Content-Disposition", "attachment; filename=\"" + fileName + ".csv\"");
//		Does not work: response.getOutputStream().write('\uFEFF'); // UTF-8 BOM for Excel
		try (CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), '\t', '"', '\\', "\n")) {
			var keysArr = new ArrayList<>();
			var row = new AtomicReference<String[]>();
			var counter = new AtomicInteger(0);

			accessionApiService.query(filterInfo.filter, select, page.toPageRequest(1000000, 10000, Sort.Direction.ASC, "seqNo"), mcpd, (one) -> {
				if (counter.getAndIncrement() == 0) {
					var row1 = (Map<?, ?>) one;
					var keys = row1.keySet();
					keysArr.addAll(keys);
					csvWriter.writeNext(row1.keySet().toArray(new String[keys.size()]), false);
					row.set(new String[keys.size()]);
				}
				var r = row.get();
				for (var i = 0; i < r.length; i++) {
					var val = one.get(keysArr.get(i));
					r[i] = val == null ? null : Objects.toString(val);
				}
				csvWriter.writeNext(r, false);
				try {
					if (counter.get() % 50 == 0) csvWriter.flush(); // Flush every 50 rows to check if the client is still connected
				} catch (IOException e) {
					throw new ClientDisconnectedException(e);
				}
			});
			csvWriter.flush();
		};
	}


	/**
	 * List distinct taxonomic data for filtered accessions
	 *
	 * @param filter the accession filter
	 * @throws IOException
	 */
	@PostMapping(value = "/species", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
	public List<Taxonomy2Info> listSpecies(@RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

		return accessionApiService.listSpecies(filter);
	}

	static interface RootNoPermissions extends JsonViews.Root, CurrentPermissionsWriter.NoPermissions { }

	/**
	 * List accessions by filterCode or filter
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws IOException
	 * @throws SearchException
	 */
	@PostMapping(value = "/images", params = { "p" }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionService.AccessionSuggestionPage<AccessionApiService.AccessionDetails, AccessionFilter> images(@RequestParam(name = "f", required = false) final String filterCode, @ParameterObject final Pagination page,
		@RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

//		FilteredPage<AccessionService.AccessionDetails> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionService.withImages(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
		FilteredPage<AccessionApiService.AccessionDetails, AccessionFilter> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.withImages(filterInfo.filter, page.toPageRequest(20, 20, Sort.Direction.ASC, "seqNo")));

		filterInfo.filter.images(true);
		Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

		return new AccessionService.AccessionSuggestionPage<>(pageRes, suggestionRes);
	}

	@PostMapping(value = "/images", params = { "o" }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionService.AccessionSuggestionSlice<AccessionApiService.AccessionDetails> images(@RequestParam(name = "f", required = false) final String filterCode,
		final ScrollPagination page, @RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		FilteredSlice<AccessionApiService.AccessionDetails> pageRes = new FilteredSlice<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.withImagesSlice(filterInfo.filter, page.toPageRequest(20, Sort.Direction.ASC, "seqNo")));

		filterInfo.filter.images(true);
		Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

		return new AccessionService.AccessionSuggestionSlice<>(pageRes, suggestionRes);
	}

	@PostMapping(value = "/images/count", produces = { MediaType.APPLICATION_JSON_VALUE })
	public int countImages(@RequestParam(name = "f", required = false) final String filterCode,
		@RequestBody(required = false) final AccessionFilter filter) throws SearchException, Exception {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		return accessionApiService.countAccessionsImages(filterInfo.filter);
	}

	@PreAuthorize("hasRole('USER')")
	@DownloadEndpoint
	@RequestMapping(value = "/downloadImages", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE, method = RequestMethod.POST)
	public void downloadImages(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, HttpServletResponse response) throws Exception {

		// get AccessionFilter from filterCode
		AccessionFilter filter = shortFilterProcessor.filterByCode(filterCode, AccessionFilter.class);
		filter.images(true);

		final long countFiltered = accessionApiService.countAccessions(filter);
		// LOG.info("Attempting to download images for {} accessions", countFiltered);
		if (countFiltered > 100) {
			throw new InvalidApiUsageException("Refusing to export more than " + 100 + " entries");
		}

		// Write Zip file archive to the stream.
		response.setBufferSize(4*1024);
		response.setContentType("application/zip");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-images-%1$s.zip\"", filterCode));

		try (final OutputStream outputStream = response.getOutputStream()) {
			try {
				downloadService.writeAccessionImageArchive(filter, outputStream);
				outputStream.flush();
				response.flushBuffer();
			} catch (Throwable e) {
				outputStream.write(e.getMessage().getBytes());
				throw e;
			}
		}
	}

	/**
	 * List accessions by filterCode or filter
	 *
	 * @param page the page
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws IOException
	 */
	@PostMapping(value = "/filter", produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionService.AccessionSuggestionPage<AccessionDTO, AccessionFilter> filter(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
		@RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

		LOG.debug("Received filter: {}", filter);
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		LOG.debug("Processed filter: {}", filterInfo.filter);
		var pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
		Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

		return new AccessionService.AccessionSuggestionPage<>(pageRes, suggestionRes);
	}

	/**
	 * Get term overview for filters
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws SearchException
	 */
	@PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
	public AccessionService.AccessionOverview overview(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter,
		@RequestParam(name = "limit", defaultValue = "10", required = false) final int limit) throws IOException, SearchException, ExecutionException, InterruptedException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		return accessionOverviewCache.get(filterInfo.filterCode + ",limit=" + limit, () -> {
			var stopWatch = StopWatch.createStarted();
			Map<String, ElasticsearchService.TermResult> overview = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, Math.min(50, limit), terms.toArray(new String[] {}));
			LOG.info("overview termStatisticsAuto {}ms", stopWatch.getTime());

			ElasticsearchService.TermResult result = elasticsearchService.recountResult(Accession.class, QAccession.accession.accessionId().storage, filterInfo.filter, overview.get("storage"), "storage");
			LOG.info("overview recountResult {}ms", stopWatch.getTime());
			overview.put("storage", result);

			long accessionCount = accessionApiService.countAccessions(filterInfo.filter);
			LOG.info("overview countAccessions {}ms", stopWatch.getTime());
			Map<String, ElasticsearchService.TermResult> suggestions = accessionApiService.getSuggestions(filterInfo.filter);
			LOG.info("overview getSuggestions {}ms", stopWatch.getTime());

			return AccessionService.AccessionOverview.from(filterInfo.filterCode, filterInfo.filter, overview, accessionCount, suggestions);
		});
	}

	/**
	 * Get overview tree for filters and provided terms
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param terms the terms (order is important!)
	 * @return the overview tree
	 *
	 * @throws SearchException the search exception
	 * @throws IOException the IO exception
	 */
	@PostMapping(value = "/overview-tree", produces = { MediaType.APPLICATION_JSON_VALUE })
	public ElasticsearchService.TreeNode overviewTree(@RequestParam(name = "f", required = false) final String filterCode,
		@RequestBody(required = false) final AccessionFilter filter,
		@RequestParam(name = "terms", required = true) final String[] terms) throws SearchException, IOException {

		if (ArrayUtils.isEmpty(terms)) {
			throw new InvalidApiUsageException("Terms must be provided!");
		}

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		var rootNode = elasticsearchService.treeNodeStatistics(Accession.class, filterInfo.filter, terms);
		rootNode.filterCode = filterInfo.filterCode;
		return rootNode;
	}

	/**
	 * Load more data for the specified term
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param term the term
	 * @return the term result
	 * @throws SearchException the search exception
	 * @throws IOException signals that an I/O exception has occurred
	 */
	@PostMapping(value = "/overview/{term}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public ElasticsearchService.TermResult loadMoreTerms(@PathVariable(name = "term") final String term, @RequestBody(required = false) final AccessionFilter filter,
		@RequestParam(name = "f", required = false) final String filterCode, @RequestParam(name = "limit", defaultValue = "20", required = false) final int limit)
		throws IOException, SearchException, ExecutionException, InterruptedException {

		if (! terms.contains(term)) {
			throw new InvalidApiUsageException("No such term. Will not search.");
		}

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		ElasticsearchService.TermResult termResult = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, Math.min(200, limit), term);
		if (term.equals("storage")) {
			termResult = elasticsearchService.recountResult(Accession.class, QAccession.accession.accessionId().storage, filterInfo.filter, termResult, "storage");
		}
		return termResult;
	}

	/**
	 * Gets accessions by list of uuid-s
	 *
	 * @param uuids accession identifi`ers to lookup in DB
	 * @return list of Accessions
	 */
	@PostMapping(value = "/for-uuid", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
	public List<AccessionDTO> forUuids(@RequestBody Set<UUID> uuids) {
		return accessionApiService.forUuids(uuids);
	}

	/**
	 * Converts AccessionIdentifiers to UUID
	 *
	 * @param identifiers accession identifiers to lookup in DB
	 * @return map with UUIDs and related AccessionIdentifiers
	 */
	@PostMapping(value = "/toUUID", produces = { MediaType.APPLICATION_JSON_VALUE })
	public Map<UUID, AccessionIdentifier3> toUUID(@RequestBody List<AccessionHeaderJson> identifiers) {
		return accessionApiService.toUUID(identifiers);
	}

	@GetMapping(value = "/details/10.{doi1:[0-9]+}/**", produces = MediaType.APPLICATION_JSON_VALUE)
	public AccessionApiService.AccessionDetails getAccessionDetailsByDoi(final HttpServletRequest request) {
		final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/details/").length());
		return accessionApiService.getAccessionDetailsByDoi(doi);
	}

	@GetMapping(value = "/details/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
	public AccessionApiService.AccessionDetails getAccessionDetailsByUUID(@PathVariable("uuid") final UUID uuid) {
		return accessionApiService.getAccessionDetailsByUuid(uuid);
	}

	@PreAuthorize("isAuthenticated()")
	@GetMapping(value = "/similar/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<DuplicateFinder.Hit<AccessionDTO>> getSimilarAccessionsForUUID(@PathVariable("uuid") final UUID uuid) {
		return accessionApiService.getSimilarAccessionsForUUID(uuid);
	}

	@PreAuthorize("isAuthenticated()")
	@PostMapping(value = "/similar/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<DuplicateFinder.Hit<AccessionDTO>> getSimilarAccessionsForUUID(@PathVariable("uuid") final UUID uuid, @RequestBody(required = false) AccessionFilter filter) {
		return accessionApiService.getSimilarAccessionsForUUID(uuid, filter);
	}

	@PreAuthorize("isAuthenticated()")
	@GetMapping(value = "/auditlog/10.{doi1:[0-9]+}/**", produces = MediaType.APPLICATION_JSON_VALUE)
	public AccessionService.AccessionAuditLog getAccessionAuditLogByDoi(final HttpServletRequest request) {
		final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/auditlog/").length());
		return accessionApiService.getAccessionAuditLogByDoi(doi);
	}

	@PreAuthorize("isAuthenticated()")
	@GetMapping(value = "/auditlog/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
	public AccessionService.AccessionAuditLog getAccessionAuditLogByUUID(@PathVariable("uuid") final UUID uuid) {
		return accessionApiService.getAccessionAuditLogByUUID(uuid);
	}

	@PostMapping(value = "/mapinfo", produces = MediaType.APPLICATION_JSON_VALUE)
	public AccessionService.AccessionMapInfo mapInfo(@RequestParam(value = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException, ExecutionException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		return accessionMapinfoCache.get(filterInfo.filterCode, () -> {

			// Force only georeferenced accessions
			AccessionFilter georefFilter = filterInfo.filter.copy(AccessionFilter.class);
			georefFilter.geo().referenced(true);

			AccessionService.AccessionMapInfo mapInfo = new AccessionService.AccessionMapInfo();
			mapInfo.filterCode = filterInfo.filterCode;
			mapInfo.filter = filterInfo.filter;

			if (StringUtils.isBlank(filterInfo.filterCode)) {
				// Entire map
				mapInfo.bounds = AccessionService.DEFAULT_GEOBOUNDS;
			} else {
				mapInfo.bounds = accessionApiService.getGeoBounds(georefFilter);
			}
			mapInfo.accessionCount = accessionApiService.countAccessions(georefFilter);
			mapInfo.tileServers = cdnServers;
			mapInfo.suggestions= accessionApiService.getSuggestions(filterInfo.filter);

			return mapInfo;
		});
	}

	/**
	 * Returns accession json by filter
	 *
	 * @param limit - max count of accession returned
	 * @param filter - filter
	 * @return json with minimal accession data
	 */
	@PostMapping(value = "/geoJson", produces = MediaType.APPLICATION_JSON_VALUE)
	public ObjectNode geoJson(@RequestParam(value = "limit", required = false, defaultValue = "100") Long limit,
		@RequestBody AccessionFilter filter) throws Exception {

		final ObjectNode geoJson = mapper.createObjectNode();
		final ArrayNode featuresArray = geoJson.arrayNode();

		accessionProcessor.process(filter, (accessions) -> {
			for (Accession accession: accessions) {
				final ObjectNode feature = featuresArray.objectNode();
				feature.put("type", "Feature");
				feature.put("id", accession.getId());

				ObjectNode geometry;
				feature.set("geometry", geometry = feature.objectNode());
				geometry.put("type", "Point");

				ArrayNode coordArray;
				geometry.set("coordinates", coordArray = geometry.arrayNode());
				coordArray.add(accession.getAccessionId().getLongitude());
				coordArray.add(accession.getAccessionId().getLatitude());

				ObjectNode properties;
				feature.set("properties", properties = feature.objectNode());
				properties.put("uuid", accession.getAccessionId().getUuid().toString());
				properties.put("doi", accession.getDoi());
				properties.put("accessionNumber", accession.getAccessionNumber());
				properties.put("instCode", accession.getInstCode());
				properties.put("datum", accession.getAccessionId().getCoordinateDatum());
				properties.put("uncertainty",  accession.getAccessionId().getCoordinateUncertainty());

				featuresArray.add(feature);
			}
		}, limit);

		geoJson.set("geoJson", featuresArray);

		long accessionCount = accessionApiService.countAccessions(filter);
		if (accessionCount > limit) {
			geoJson.put("otherCount", accessionCount - limit);
		}

		return geoJson;
	}

	@GetMapping(value = "/autocomplete/{field:.+}", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<AccessionService.LabelValue<String>> autocomplete(@PathVariable("field") String field,
		@RequestParam(value = "term", required = true) String term,
		@RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		return accessionApiService.autocomplete(filterInfo.filter, field, term);
	}

	@DownloadEndpoint
	@RequestMapping(value = "/downloadKml", produces = "application/vnd.google-earth.kml+xml", method = RequestMethod.POST)
	public void downloadKml(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws Exception {

		// get AccessionFilter from filterCode
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		AccessionGeoFilter geoFilter = filterInfo.filter.geo;
		if (geoFilter == null) {
			filterInfo.filter.geo = geoFilter = new AccessionGeoFilter();
		}
		geoFilter.referenced(true);

		final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
		// LOG.info("Attempting to download KML for {} accessions", countFiltered);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		response.setContentType("application/vnd.google-earth.kml+xml");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-kml-%1s.kml\"", filterInfo.filterCode));

		DecimalFormatSymbols dfs = new DecimalFormatSymbols();
		dfs.setDecimalSeparator('.');
		DecimalFormat decimalFormat = new DecimalFormat("0.#", dfs);
		decimalFormat.setMinimumIntegerDigits(1);
		decimalFormat.setMinimumFractionDigits(6);
		decimalFormat.setGroupingUsed(false);

		// Write KML to the stream.
		final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()));
		writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
		writer.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n");
		writer.write("<Document>\n");
		try {
			accessionProcessor.process(filterInfo.filter, (accessions) -> {
				for (Accession accession: accessions) {
					AccessionId aid = accession.getAccessionId();
					if (aid != null && aid.getLongitude() != null && aid.getLatitude() != null) {
						writer.append("<Placemark>");
						writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");

						writer.append("<description><![CDATA[\n");
						writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
						writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
						writer.append("<p><a href=\"").append(frontendUrl).append("/a/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
						writer.append("\n]]></description>");

						writer.append("<Point><coordinates>");
						writer.append(decimalFormat.format(aid.getLongitude())).append(",").append(decimalFormat.format(aid.getLatitude()));
						writer.append("</coordinates></Point>");
						writer.append("</Placemark>\n");
						writer.flush();
					}
				}
			});
			writer.write("</Document>\n</kml>\n");

		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		} catch (Exception e) {
			LOG.warn("Error generating KML: {}", e.getMessage());
			throw e;
		} finally {
			writer.flush();
			writer.close();
			response.flushBuffer();
		}
	}

	@DownloadEndpoint
	@RequestMapping(value = "/download-selected", method = RequestMethod.POST, params = { "mcpd" })
	public void downloadMcpdByUuids(@RequestParam(value="uuids", required = true) Set<UUID> uuids, HttpServletResponse response) throws IOException, SearchException {
		// Create JSON filter
		AccessionFilter filter = new AccessionFilter();
		filter.historic(null);
		filter.uuid(new HashSet<>(uuids));

		final long countFiltered = accessionApiService.countAccessions(filter);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		// Write MCPD to the stream.
		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-selected-%1s.xlsx\"", System.currentTimeMillis()));

		final OutputStream outputStream = response.getOutputStream();
		try {
			downloadService.writeXlsxMCPD(filter, outputStream, null, "/sel");
			response.flushBuffer();
		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		}
	}

	@DownloadEndpoint
	@RequestMapping(value = "/download-selected", method = RequestMethod.POST, params = { "dwca" })
	public void downloadDwcaByUuids(@RequestParam(value="uuids", required = true) Set<UUID> uuids, HttpServletResponse response) throws Exception {
		// Create JSON filter
		AccessionFilter filter = new AccessionFilter();
		filter
			.historic(null)
			.uuid(new HashSet<>(uuids));

		final long countFiltered = accessionApiService.countAccessions(filter);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		// Write Darwin Core Archive to the stream.
		response.setContentType("application/zip");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-selected-%1$s.zip\"", System.currentTimeMillis()));

		final OutputStream outputStream = response.getOutputStream();
		genesysService.writeAccessions(filter, outputStream, null, "/sel");
		response.flushBuffer();
	}

	@DownloadEndpoint
	@RequestMapping(value = "/download", method = RequestMethod.POST, params = { "mcpd" })
	public void downloadMcpd(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws IOException, SearchException {

		// get AccessionFilter from filterCode
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		// Write MCPD to the stream.
		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1s.xlsx\"", filterInfo.filterCode));
		// response.flushBuffer();

		final OutputStream outputStream = response.getOutputStream();
		try {
			downloadService.writeXlsxMCPD(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
			response.flushBuffer();
		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		}
	}


	@DownloadEndpoint
	@RequestMapping(value = "/download", method = RequestMethod.POST, params = { "pdci" })
	public void downloadPdci(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws IOException, SearchException {

		// get AccessionFilter from filterCode
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		// Write PDCI to the stream.
		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-PDCI-%1s.xlsx\"", filterInfo.filterCode));
		// response.flushBuffer();

		final OutputStream outputStream = response.getOutputStream();
		try {
			downloadService.writeXlsxPDCI(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
			response.flushBuffer();
		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		}
	}

	@DownloadEndpoint
	@RequestMapping(value = "/download", method = RequestMethod.POST, params = { "dwca" })
	public void downloadDwca(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws Exception {

		// get AccessionFilter from filterCode
		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

		final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
		if (countFiltered > DOWNLOAD_LIMIT) {
			throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
		}

		// Write Darwin Core Archive to the stream.
		response.setContentType("application/zip");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1$s.zip\"", filterInfo.filterCode));

		final OutputStream outputStream = response.getOutputStream();
		genesysService.writeAccessions(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
		response.flushBuffer();
	}


	/**
	 * Returns accession json by filter
	 *
	 * @param params - similarity search params {@link org.genesys.server.api.v1.AccessionController.SimilaritySearchParams}
	 * @return json with minimal accession data
	 */
	@PostMapping(value = "/find-similar", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<DuplicateFinder.SimilarityHit<Accession>> findSimilar(@RequestBody org.genesys.server.api.v1.AccessionController.SimilaritySearchParams params) throws Exception {
		List<DuplicateFinder.SimilarityHit<Accession>> results = new ArrayList<>();

		final long countFiltered = accessionApiService.countAccessions(params.select);

		if (countFiltered > 100) {
			throw new InvalidApiUsageException("Too many matches for similarity search!");
		}

		accessionProcessor.process(params.select, (accessions) -> {
			results.addAll(duplicateFinder.findSimilar(accessions, params.target));
		});

		return results;
	}

	@Accessors(fluent = true, chain = true)
	@Getter
	@Setter
	public static class SimilaritySearchParams {
		public AccessionFilter select; // Which accessions to process
		public AccessionFilter target; // What target filter to apply
	}

	/**
	 * Get term overview for filters
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws Exception
	 */
	@PostMapping(value = "/tileIndex3min", produces = { MediaType.APPLICATION_JSON_VALUE })
	public Set<Integer> allTileIndex3min(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter) throws Exception {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		return accessionApiService.listTileIndex3min(filterInfo.filter);
	}

	/**
	 * Get term overview for filters
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the page
	 * @throws Exception
	 */
	@PostMapping(value = "/tileIndex3min", produces = { MediaType.APPLICATION_JSON_VALUE }, params = { "crop" })
	public Map<String, Set<Integer>> allTileIndex3minByCrop(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter) throws Exception {

		ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
		return accessionApiService.listTileIndex3minByCrop(filterInfo.filter);
	}

	@GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/observations", produces = MediaType.APPLICATION_JSON_VALUE)
	public AmphibianService.AccessionObservations getAccessionObservations(@PathVariable("uuid") final UUID uuid) throws Exception {
		return amphibianService.getAccessionObservations(uuid);
	}
}