AccessionController.java

/*
 * Copyright 2024 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.api.v2.impl;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
import com.opencsv.CSVWriter;
import io.swagger.annotations.Api;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.blocks.model.JsonViews;
import org.genesys.blocks.security.serialization.CurrentPermissionsWriter;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.FilteredPage;
import org.genesys.server.api.FilteredSlice;
import org.genesys.server.api.Pagination;
import org.genesys.server.api.ScrollPagination;
import org.genesys.server.api.model.AccessionHeaderJson;
import org.genesys.server.api.v2.facade.AccessionApiService;
import org.genesys.server.api.v2.model.impl.AccessionDTO;
import org.genesys.server.api.v2.model.impl.Taxonomy2Info;
import org.genesys.server.component.aspect.DownloadEndpoint;
import org.genesys.server.exception.ClientDisconnectedException;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.genesys.AccessionId;
import org.genesys.server.model.genesys.QAccession;
import org.genesys.server.model.impl.AccessionIdentifier3;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.AmphibianService;
import org.genesys.server.service.DownloadService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.ShortFilterService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.service.filter.AccessionGeoFilter;
import org.genesys.server.service.impl.AmphibianServiceImpl;
import org.genesys.server.service.worker.AccessionProcessor;
import org.genesys.server.service.worker.ShortFilterProcessor;
import org.genesys.server.service.worker.dupe.AccessionDuplicateFinder;
import org.genesys.server.service.worker.dupe.DuplicateFinder;
import org.genesys.spring.CSVMessageConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springdoc.api.annotations.ParameterObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.HandlerMapping;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedWriter;
import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

/**
 * Accession API v2
 */
@RestController("accessionApi2")
@PreAuthorize("isAuthenticated()")
@RequestMapping(AccessionController.CONTROLLER_URL)
@Api(tags = { "accession" })
public class AccessionController extends ApiBaseController {

    private static final Logger LOG = LoggerFactory.getLogger(AccessionController.class);

    /** The Constant CONTROLLER_URL. */
    public static final String CONTROLLER_URL = ApiBaseController.APIv2_BASE + "/acn";

    private static final int DOWNLOAD_LIMIT = 300000;

    @Autowired
    private AccessionApiService accessionApiService;

    @Autowired
    protected ShortFilterProcessor shortFilterProcessor;

    @Autowired(required = false)
    private ElasticsearchService elasticsearchService;

    @Autowired
    private DownloadService downloadService;

    @Autowired
    private GenesysService genesysService;

    @Autowired
    private AmphibianService amphibianService;

    @Autowired
    private AccessionProcessor accessionProcessor;

    @Autowired
    private ObjectMapper objectMapper;

    private final Cache<String, AccessionService.AccessionOverview> accessionOverviewCache = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(5, TimeUnit.MINUTES).build();
    private final Cache<String, AccessionService.AccessionMapInfo> accessionMapinfoCache = CacheBuilder.newBuilder().maximumSize(50).expireAfterWrite(5, TimeUnit.MINUTES).build();

    private final ObjectMapper mapper = new ObjectMapper();

    @Value("${frontend.url}")
    private String frontendUrl;

    @Value("${cdn.servers}")
    private String[] cdnServers;

    public static final Set<String> terms = Sets.newHashSet("institute.code", "institute.country.code3", "cropName", "crop.shortName", "taxonomy.genus", "taxonomy.species",
        "taxonomy.genusSpecies", "taxonomy.grinTaxonomySpecies.name", "taxonomy.currentTaxonomySpecies.name", "countryOfOrigin.code3", "sampStat", "available", "mlsStatus",
        "donorCode", "sgsv", "storage", "duplSite", "breederCode", "aegis", "curationType");

    @Autowired
    private AccessionDuplicateFinder duplicateFinder;

    @GetMapping(value = "/id/{id}", produces = { MediaType.APPLICATION_JSON_VALUE })
    public UUID uuidFromId(@PathVariable("id") final long id) {
        return accessionApiService.uuidFromId(id);
    }

    @PostMapping(value = "/id", produces = { MediaType.APPLICATION_JSON_VALUE })
    public List<UUID> uuidFromIds(@RequestBody List<Long> ids) {
        return accessionApiService.uuidsFromIds(ids);
    }


    @GetMapping(value = "/acce-number/{acceNumber}", produces = { MediaType.APPLICATION_JSON_VALUE })
    public UUID uuidFromAcceNumber(@RequestParam(value = "instCode", required = false) String instCode , @PathVariable("acceNumber") String acceNumber) {
        return accessionApiService.uuidFromAcceNumber(instCode, acceNumber);
    }

    @PostMapping(value = "/acce-number", produces = { MediaType.APPLICATION_JSON_VALUE })
    public List<UUID> uuidsFromAcceNumbers(@RequestParam(value = "instCode", required = false) String instCode, @RequestBody List<String> acceNumbers) {
        return accessionApiService.uuidsFromAcceNumbers(instCode, acceNumbers);
    }

    /**
     * Gets the accession
     *
     * @param uuid the uuid
     * @return the subset
     */
    @GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}", produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionDTO getByUuid(@PathVariable("uuid") final UUID uuid) {
        return accessionApiService.getByUuid(uuid);
    }

    /**
     * Gets the accession
     *
     * @return the subset
     */
    @GetMapping(value = "/10.{doi1:[0-9]+}/**", produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionDTO getByDoi(final HttpServletRequest request) {
        final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/").length());
        return accessionApiService.getByDoi(doi);
    }

    /**
     * List accessions by filterCode or filter
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws IOException
     */
    @PostMapping(value = "/list", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
    public FilteredPage<AccessionDTO, AccessionFilter> list(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
        @RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
    }


    /**
     * Query accession fields 
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @param select list of {@code Accession} fields to include in the response
     * @param mcpd Concatenate arrays with ";". Default is <b>{@code false}</b>
     * @return the page
     * @throws Exception
     */
    @PostMapping(value = "/query", produces = { MediaType.APPLICATION_JSON_VALUE })
    public FilteredPage<?, AccessionFilter> query(
        final Pagination page,
        @RequestParam(name = "f", required = false) String filterCode,
        @RequestBody(required = false) AccessionFilter filter,
        @RequestParam List<String> select,
        @RequestParam(required = false, defaultValue = "false") boolean mcpd,
        HttpServletRequest request
    ) throws Exception {
        LOG.warn("Using MCPD style? {}", mcpd);
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
//      return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionService.query(filterInfo.filter, select, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE), mcpd));
        return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.query(filterInfo.filter, select, page.toPageRequest(10000, DEFAULT_PAGE_SIZE), mcpd));
    }

    /**
     * Query accession fields (for HTTP Form POST)
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @param select list of {@code Accession} fields to include in the response
     * @param mcpd Concatenate arrays with ";". Default is <b>{@code true}</b>
     * @throws Exception
     */
    @PostMapping(value = "/query", produces = { CSVMessageConverter.TEXT_CSV_VALUE, CSVMessageConverter.TEXT_TSV_VALUE }, consumes = { MediaType.APPLICATION_FORM_URLENCODED_VALUE, MediaType.MULTIPART_FORM_DATA_VALUE })
    public void queryCsvForm(
        final Pagination page,
        @RequestParam(name = "f", required = false) String filterCode,
        @RequestParam String filter,
        @RequestParam List<String> select,
        @RequestParam(required = false, defaultValue = "true") boolean mcpd,
        HttpServletRequest request,
        HttpServletResponse response
    ) throws Exception {
        queryCsv(page, filterCode, objectMapper.readValue(filter, AccessionFilter.class), select, mcpd, request, response);
    }
    /**
     * Query accession fields. 
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @param select list of {@code Accession} fields to include in the response
     * @param mcpd Concatenate arrays with ";". Default is <b>{@code true}</b>
     * @throws Exception
     */
    @PostMapping(value = "/query", produces = { CSVMessageConverter.TEXT_CSV_VALUE, CSVMessageConverter.TEXT_TSV_VALUE })
    public void queryCsv(
        final Pagination page,
        @RequestParam(name = "f", required = false) String filterCode,
        @RequestBody(required = false) AccessionFilter filter,
        @RequestParam List<String> select,
        @RequestParam(required = false, defaultValue = "true") boolean mcpd,
        HttpServletRequest request,
        HttpServletResponse response
    ) throws Exception {
        LOG.debug("Using MCPD style? {}", mcpd);
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        response.addHeader("Content-Type", CSVMessageConverter.TEXT_TSV_VALUE);
        String fileName = "query" + (StringUtils.isNotBlank(filterInfo.filterCode) ? "-".concat(filterInfo.filterCode) : "");
        response.addHeader("Content-Disposition", "attachment; filename=\"" + fileName + ".csv\"");
//      Does not work: response.getOutputStream().write('\uFEFF'); // UTF-8 BOM for Excel
        try (CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(response.getOutputStream(), "UTF8"), '\t', '"', '\\', "\n")) {
            var keysArr = new ArrayList<>();
            var row = new AtomicReference<String[]>();
            var counter = new AtomicInteger(0);

            accessionApiService.query(filterInfo.filter, select, page.toPageRequest(1000000, 10000, Sort.Direction.ASC, "seqNo"), mcpd, (one) -> {
                if (counter.getAndIncrement() == 0) {
                    var row1 = (Map<?, ?>) one;
                    var keys = row1.keySet();
                    keysArr.addAll(keys);
                    csvWriter.writeNext(row1.keySet().toArray(new String[keys.size()]), false);
                    row.set(new String[keys.size()]);
                }
                var r = row.get();
                for (var i = 0; i < r.length; i++) {
                    var val = one.get(keysArr.get(i));
                    r[i] = val == null ? null : Objects.toString(val);
                }
                csvWriter.writeNext(r, false);
                try {
                    if (counter.get() % 50 == 0) csvWriter.flush(); // Flush every 50 rows to check if the client is still connected
                } catch (IOException e) {
                    throw new ClientDisconnectedException(e);
                }
            });
            csvWriter.flush();
        };
    }


    /**
     * List distinct taxonomic data for filtered accessions
     *
     * @param filter the accession filter
     * @throws IOException
     */
    @PostMapping(value = "/species", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
    public List<Taxonomy2Info> listSpecies(@RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

        return accessionApiService.listSpecies(filter);
    }

    static interface RootNoPermissions extends JsonViews.Root, CurrentPermissionsWriter.NoPermissions { }

    /**
     * List accessions by filterCode or filter
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws IOException
     * @throws SearchException
     */
    @PostMapping(value = "/images", params = { "p" }, produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionService.AccessionSuggestionPage<AccessionApiService.AccessionDetails, AccessionFilter> images(@RequestParam(name = "f", required = false) final String filterCode, @ParameterObject final Pagination page,
        @RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

//      FilteredPage<AccessionService.AccessionDetails> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionService.withImages(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
        FilteredPage<AccessionApiService.AccessionDetails, AccessionFilter> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.withImages(filterInfo.filter, page.toPageRequest(20, 20, Sort.Direction.ASC, "seqNo")));

        filterInfo.filter.images(true);
        Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

        return new AccessionService.AccessionSuggestionPage<>(pageRes, suggestionRes);
    }

    @PostMapping(value = "/images", params = { "o" }, produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionService.AccessionSuggestionSlice<AccessionApiService.AccessionDetails> images(@RequestParam(name = "f", required = false) final String filterCode,
        final ScrollPagination page, @RequestBody(required = false) final AccessionFilter filter) throws IOException, SearchException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        FilteredSlice<AccessionApiService.AccessionDetails> pageRes = new FilteredSlice<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.withImagesSlice(filterInfo.filter, page.toPageRequest(20, Sort.Direction.ASC, "seqNo")));

        filterInfo.filter.images(true);
        Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

        return new AccessionService.AccessionSuggestionSlice<>(pageRes, suggestionRes);
    }

    @PostMapping(value = "/images/count", produces = { MediaType.APPLICATION_JSON_VALUE })
    public int countImages(@RequestParam(name = "f", required = false) final String filterCode,
        @RequestBody(required = false) final AccessionFilter filter) throws SearchException, Exception {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        return accessionApiService.countAccessionsImages(filterInfo.filter);
    }

    @PreAuthorize("hasRole('USER')")
    @DownloadEndpoint
    @RequestMapping(value = "/downloadImages", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE, method = RequestMethod.POST)
    public void downloadImages(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, HttpServletResponse response) throws Exception {

        // get AccessionFilter from filterCode
        AccessionFilter filter = shortFilterProcessor.filterByCode(filterCode, AccessionFilter.class);
        filter.images(true);

        final long countFiltered = accessionApiService.countAccessions(filter);
        // LOG.info("Attempting to download images for {} accessions", countFiltered);
        if (countFiltered > 100) {
            throw new InvalidApiUsageException("Refusing to export more than " + 100 + " entries");
        }

        // Write Zip file archive to the stream.
        response.setBufferSize(4*1024);
        response.setContentType("application/zip");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-images-%1$s.zip\"", filterCode));

        try (final OutputStream outputStream = response.getOutputStream()) {
            try {
                downloadService.writeAccessionImageArchive(filter, outputStream);
                outputStream.flush();
                response.flushBuffer();
            } catch (Throwable e) {
                outputStream.write(e.getMessage().getBytes());
                throw e;
            }
        }
    }

    /**
     * List accessions by filterCode or filter
     *
     * @param page the page
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws IOException
     */
    @PostMapping(value = "/filter", produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionService.AccessionSuggestionPage<AccessionDTO, AccessionFilter> filter(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
        @RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException {

        LOG.debug("Received filter: {}", filter);
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        LOG.debug("Processed filter: {}", filterInfo.filter);
        var pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, accessionApiService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "seqNo")));
        Map<String, ElasticsearchService.TermResult> suggestionRes = accessionApiService.getSuggestions(filterInfo.filter);

        return new AccessionService.AccessionSuggestionPage<>(pageRes, suggestionRes);
    }

    /**
     * Get term overview for filters
     *
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws SearchException
     */
    @PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
    public AccessionService.AccessionOverview overview(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter,
        @RequestParam(name = "limit", defaultValue = "10", required = false) final int limit) throws IOException, SearchException, ExecutionException, InterruptedException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        return accessionOverviewCache.get(filterInfo.filterCode + ",limit=" + limit, () -> {
            var stopWatch = StopWatch.createStarted();
            Map<String, ElasticsearchService.TermResult> overview = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, Math.min(50, limit), terms.toArray(new String[] {}));
            LOG.info("overview termStatisticsAuto {}ms", stopWatch.getTime());

            ElasticsearchService.TermResult result = elasticsearchService.recountResult(Accession.class, QAccession.accession.accessionId().storage, filterInfo.filter, overview.get("storage"), "storage");
            LOG.info("overview recountResult {}ms", stopWatch.getTime());
            overview.put("storage", result);

            long accessionCount = accessionApiService.countAccessions(filterInfo.filter);
            LOG.info("overview countAccessions {}ms", stopWatch.getTime());
            Map<String, ElasticsearchService.TermResult> suggestions = accessionApiService.getSuggestions(filterInfo.filter);
            LOG.info("overview getSuggestions {}ms", stopWatch.getTime());

            return AccessionService.AccessionOverview.from(filterInfo.filterCode, filterInfo.filter, overview, accessionCount, suggestions);
        });
    }

    /**
     * Get overview tree for filters and provided terms
     *
     * @param filterCode short filter code
     * @param filter the filter
     * @param terms the terms (order is important!)
     * @return the overview tree
     *
     * @throws SearchException the search exception
     * @throws IOException the IO exception
     */
    @PostMapping(value = "/overview-tree", produces = { MediaType.APPLICATION_JSON_VALUE })
    public ElasticsearchService.TreeNode overviewTree(@RequestParam(name = "f", required = false) final String filterCode,
        @RequestBody(required = false) final AccessionFilter filter,
        @RequestParam(name = "terms", required = true) final String[] terms) throws SearchException, IOException {

        if (ArrayUtils.isEmpty(terms)) {
            throw new InvalidApiUsageException("Terms must be provided!");
        }

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        var rootNode = elasticsearchService.treeNodeStatistics(Accession.class, filterInfo.filter, terms);
        rootNode.filterCode = filterInfo.filterCode;
        return rootNode;
    }

    /**
     * Load more data for the specified term
     *
     * @param filterCode short filter code
     * @param filter the filter
     * @param term the term
     * @return the term result
     * @throws SearchException the search exception
     * @throws IOException signals that an I/O exception has occurred
     */
    @PostMapping(value = "/overview/{term}", produces = { MediaType.APPLICATION_JSON_VALUE })
    public ElasticsearchService.TermResult loadMoreTerms(@PathVariable(name = "term") final String term, @RequestBody(required = false) final AccessionFilter filter,
        @RequestParam(name = "f", required = false) final String filterCode, @RequestParam(name = "limit", defaultValue = "20", required = false) final int limit)
        throws IOException, SearchException, ExecutionException, InterruptedException {

        if (! terms.contains(term)) {
            throw new InvalidApiUsageException("No such term. Will not search.");
        }

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        ElasticsearchService.TermResult termResult = elasticsearchService.termStatisticsAuto(Accession.class, filterInfo.filter, Math.min(200, limit), term);
        if (term.equals("storage")) {
            termResult = elasticsearchService.recountResult(Accession.class, QAccession.accession.accessionId().storage, filterInfo.filter, termResult, "storage");
        }
        return termResult;
    }

    /**
     * Gets accessions by list of uuid-s
     *
     * @param uuids accession identifi`ers to lookup in DB
     * @return list of Accessions
     */
    @PostMapping(value = "/for-uuid", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
    public List<AccessionDTO> forUuids(@RequestBody Set<UUID> uuids) {
        return accessionApiService.forUuids(uuids);
    }

    /**
     * Converts AccessionIdentifiers to UUID
     *
     * @param identifiers accession identifiers to lookup in DB
     * @return map with UUIDs and related AccessionIdentifiers
     */
    @PostMapping(value = "/toUUID", produces = { MediaType.APPLICATION_JSON_VALUE })
    public Map<UUID, AccessionIdentifier3> toUUID(@RequestBody List<AccessionHeaderJson> identifiers) {
        return accessionApiService.toUUID(identifiers);
    }

    @GetMapping(value = "/details/10.{doi1:[0-9]+}/**", produces = MediaType.APPLICATION_JSON_VALUE)
    public AccessionApiService.AccessionDetails getAccessionDetailsByDoi(final HttpServletRequest request) {
        final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/details/").length());
        return accessionApiService.getAccessionDetailsByDoi(doi);
    }

    @GetMapping(value = "/details/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
    public AccessionApiService.AccessionDetails getAccessionDetailsByUUID(@PathVariable("uuid") final UUID uuid) {
        return accessionApiService.getAccessionDetailsByUuid(uuid);
    }

    @PreAuthorize("isAuthenticated()")
    @GetMapping(value = "/similar/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
    public List<DuplicateFinder.Hit<AccessionDTO>> getSimilarAccessionsForUUID(@PathVariable("uuid") final UUID uuid) {
        return accessionApiService.getSimilarAccessionsForUUID(uuid);
    }

    @PreAuthorize("isAuthenticated()")
    @PostMapping(value = "/similar/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
    public List<DuplicateFinder.Hit<AccessionDTO>> getSimilarAccessionsForUUID(@PathVariable("uuid") final UUID uuid, @RequestBody(required = false) AccessionFilter filter) {
        return accessionApiService.getSimilarAccessionsForUUID(uuid, filter);
    }

    @PreAuthorize("isAuthenticated()")
    @GetMapping(value = "/auditlog/10.{doi1:[0-9]+}/**", produces = MediaType.APPLICATION_JSON_VALUE)
    public AccessionService.AccessionAuditLog getAccessionAuditLogByDoi(final HttpServletRequest request) {
        final String doi = ((String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE)).substring((CONTROLLER_URL + "/auditlog/").length());
        return accessionApiService.getAccessionAuditLogByDoi(doi);
    }

    @PreAuthorize("isAuthenticated()")
    @GetMapping(value = "/auditlog/{uuid:\\w{8}\\-\\w{4}.+}", produces = MediaType.APPLICATION_JSON_VALUE)
    public AccessionService.AccessionAuditLog getAccessionAuditLogByUUID(@PathVariable("uuid") final UUID uuid) {
        return accessionApiService.getAccessionAuditLogByUUID(uuid);
    }

    @PostMapping(value = "/mapinfo", produces = MediaType.APPLICATION_JSON_VALUE)
    public AccessionService.AccessionMapInfo mapInfo(@RequestParam(value = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException, SearchException, ExecutionException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        return accessionMapinfoCache.get(filterInfo.filterCode, () -> {

            // Force only georeferenced accessions
            AccessionFilter georefFilter = filterInfo.filter.copy(AccessionFilter.class);
            georefFilter.geo().referenced(true);

            AccessionService.AccessionMapInfo mapInfo = new AccessionService.AccessionMapInfo();
            mapInfo.filterCode = filterInfo.filterCode;
            mapInfo.filter = filterInfo.filter;

            if (StringUtils.isBlank(filterInfo.filterCode)) {
                // Entire map
                mapInfo.bounds = AccessionService.DEFAULT_GEOBOUNDS;
            } else {
                mapInfo.bounds = accessionApiService.getGeoBounds(georefFilter);
            }
            mapInfo.accessionCount = accessionApiService.countAccessions(georefFilter);
            mapInfo.tileServers = cdnServers;
            mapInfo.suggestions= accessionApiService.getSuggestions(filterInfo.filter);

            return mapInfo;
        });
    }

    /**
     * Returns accession json by filter
     *
     * @param limit - max count of accession returned
     * @param filter - filter
     * @return json with minimal accession data
     */
    @PostMapping(value = "/geoJson", produces = MediaType.APPLICATION_JSON_VALUE)
    public ObjectNode geoJson(@RequestParam(value = "limit", required = false, defaultValue = "100") Long limit,
        @RequestBody AccessionFilter filter) throws Exception {

        final ObjectNode geoJson = mapper.createObjectNode();
        final ArrayNode featuresArray = geoJson.arrayNode();

        accessionProcessor.process(filter, (accessions) -> {
            for (Accession accession: accessions) {
                final ObjectNode feature = featuresArray.objectNode();
                feature.put("type", "Feature");
                feature.put("id", accession.getId());

                ObjectNode geometry;
                feature.set("geometry", geometry = feature.objectNode());
                geometry.put("type", "Point");

                ArrayNode coordArray;
                geometry.set("coordinates", coordArray = geometry.arrayNode());
                coordArray.add(accession.getAccessionId().getLongitude());
                coordArray.add(accession.getAccessionId().getLatitude());

                ObjectNode properties;
                feature.set("properties", properties = feature.objectNode());
                properties.put("uuid", accession.getAccessionId().getUuid().toString());
                properties.put("doi", accession.getDoi());
                properties.put("accessionNumber", accession.getAccessionNumber());
                properties.put("instCode", accession.getInstCode());
                properties.put("datum", accession.getAccessionId().getCoordinateDatum());
                properties.put("uncertainty",  accession.getAccessionId().getCoordinateUncertainty());

                featuresArray.add(feature);
            }
        }, limit);

        geoJson.set("geoJson", featuresArray);

        long accessionCount = accessionApiService.countAccessions(filter);
        if (accessionCount > limit) {
            geoJson.put("otherCount", accessionCount - limit);
        }

        return geoJson;
    }

    @GetMapping(value = "/autocomplete/{field:.+}", produces = MediaType.APPLICATION_JSON_VALUE)
    public List<AccessionService.LabelValue<String>> autocomplete(@PathVariable("field") String field,
        @RequestParam(value = "term", required = true) String term,
        @RequestParam(name = "f", required = false) String filterCode, @RequestBody(required = false) AccessionFilter filter) throws IOException {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        return accessionApiService.autocomplete(filterInfo.filter, field, term);
    }

    @DownloadEndpoint
    @RequestMapping(value = "/downloadKml", produces = "application/vnd.google-earth.kml+xml", method = RequestMethod.POST)
    public void downloadKml(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws Exception {

        // get AccessionFilter from filterCode
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        AccessionGeoFilter geoFilter = filterInfo.filter.geo;
        if (geoFilter == null) {
            filterInfo.filter.geo = geoFilter = new AccessionGeoFilter();
        }
        geoFilter.referenced(true);

        final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
        // LOG.info("Attempting to download KML for {} accessions", countFiltered);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        response.setContentType("application/vnd.google-earth.kml+xml");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-kml-%1s.kml\"", filterInfo.filterCode));

        DecimalFormatSymbols dfs = new DecimalFormatSymbols();
        dfs.setDecimalSeparator('.');
        DecimalFormat decimalFormat = new DecimalFormat("0.#", dfs);
        decimalFormat.setMinimumIntegerDigits(1);
        decimalFormat.setMinimumFractionDigits(6);
        decimalFormat.setGroupingUsed(false);

        // Write KML to the stream.
        final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(response.getOutputStream()));
        writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
        writer.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n");
        writer.write("<Document>\n");
        try {
            accessionProcessor.process(filterInfo.filter, (accessions) -> {
                for (Accession accession: accessions) {
                    AccessionId aid = accession.getAccessionId();
                    if (aid != null && aid.getLongitude() != null && aid.getLatitude() != null) {
                        writer.append("<Placemark>");
                        writer.append("<name>").append(accession.getAccessionNumber()).append("</name>");

                        writer.append("<description><![CDATA[\n");
                        writer.append("<p>").append(accession.getTaxonomy().getTaxonNameHtml()).append("</p>");
                        writer.append("<p>").append(accession.getInstitute().getCode()).append(" ").append(accession.getInstitute().getFullName()).append("</p>");
                        writer.append("<p><a href=\"").append(frontendUrl).append("/a/").append(accession.getUuid().toString()).append("\">Passport data</a></p>");
                        writer.append("\n]]></description>");

                        writer.append("<Point><coordinates>");
                        writer.append(decimalFormat.format(aid.getLongitude())).append(",").append(decimalFormat.format(aid.getLatitude()));
                        writer.append("</coordinates></Point>");
                        writer.append("</Placemark>\n");
                        writer.flush();
                    }
                }
            });
            writer.write("</Document>\n</kml>\n");

        } catch (EOFException e) {
            LOG.warn("Download was aborted: {}", e.getMessage());
            throw e;
        } catch (Exception e) {
            LOG.warn("Error generating KML: {}", e.getMessage());
            throw e;
        } finally {
            writer.flush();
            writer.close();
            response.flushBuffer();
        }
    }

    @DownloadEndpoint
    @RequestMapping(value = "/download-selected", method = RequestMethod.POST, params = { "mcpd" })
    public void downloadMcpdByUuids(@RequestParam(value="uuids", required = true) Set<UUID> uuids, HttpServletResponse response) throws IOException, SearchException {
        // Create JSON filter
        AccessionFilter filter = new AccessionFilter();
        filter.historic(null);
        filter.uuid(new HashSet<>(uuids));

        final long countFiltered = accessionApiService.countAccessions(filter);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        // Write MCPD to the stream.
        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-selected-%1s.xlsx\"", System.currentTimeMillis()));

        final OutputStream outputStream = response.getOutputStream();
        try {
            downloadService.writeXlsxMCPD(filter, outputStream, null, "/sel");
            response.flushBuffer();
        } catch (EOFException e) {
            LOG.warn("Download was aborted: {}", e.getMessage());
            throw e;
        }
    }

    @DownloadEndpoint
    @RequestMapping(value = "/download-selected", method = RequestMethod.POST, params = { "dwca" })
    public void downloadDwcaByUuids(@RequestParam(value="uuids", required = true) Set<UUID> uuids, HttpServletResponse response) throws Exception {
        // Create JSON filter
        AccessionFilter filter = new AccessionFilter();
        filter
            .historic(null)
            .uuid(new HashSet<>(uuids));

        final long countFiltered = accessionApiService.countAccessions(filter);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        // Write Darwin Core Archive to the stream.
        response.setContentType("application/zip");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-selected-%1$s.zip\"", System.currentTimeMillis()));

        final OutputStream outputStream = response.getOutputStream();
        genesysService.writeAccessions(filter, outputStream, null, "/sel");
        response.flushBuffer();
    }

    @DownloadEndpoint
    @RequestMapping(value = "/download", method = RequestMethod.POST, params = { "mcpd" })
    public void downloadMcpd(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws IOException, SearchException {

        // get AccessionFilter from filterCode
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        // Write MCPD to the stream.
        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1s.xlsx\"", filterInfo.filterCode));
        // response.flushBuffer();

        final OutputStream outputStream = response.getOutputStream();
        try {
            downloadService.writeXlsxMCPD(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
            response.flushBuffer();
        } catch (EOFException e) {
            LOG.warn("Download was aborted: {}", e.getMessage());
            throw e;
        }
    }


    @DownloadEndpoint
    @RequestMapping(value = "/download", method = RequestMethod.POST, params = { "pdci" })
    public void downloadPdci(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws IOException, SearchException {

        // get AccessionFilter from filterCode
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        // Write PDCI to the stream.
        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-PDCI-%1s.xlsx\"", filterInfo.filterCode));
        // response.flushBuffer();

        final OutputStream outputStream = response.getOutputStream();
        try {
            downloadService.writeXlsxPDCI(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
            response.flushBuffer();
        } catch (EOFException e) {
            LOG.warn("Download was aborted: {}", e.getMessage());
            throw e;
        }
    }

    @DownloadEndpoint
    @RequestMapping(value = "/download", method = RequestMethod.POST, params = { "dwca" })
    public void downloadDwca(@RequestParam(value = "f", required = false, defaultValue = "") String filterCode, @RequestParam(value="filter", required = false) AccessionFilter filter, HttpServletResponse response) throws Exception {

        // get AccessionFilter from filterCode
        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);

        final long countFiltered = accessionApiService.countAccessions(filterInfo.filter);
        if (countFiltered > DOWNLOAD_LIMIT) {
            throw new InvalidApiUsageException("Refusing to export more than " + DOWNLOAD_LIMIT + " entries");
        }

        // Write Darwin Core Archive to the stream.
        response.setContentType("application/zip");
        response.addHeader("Content-Disposition", String.format("attachment; filename=\"genesys-accessions-%1$s.zip\"", filterInfo.filterCode));

        final OutputStream outputStream = response.getOutputStream();
        genesysService.writeAccessions(filterInfo.filter, outputStream, filterInfo.filterCode, "/a/" + filterInfo.filterCode);
        response.flushBuffer();
    }


    /**
     * Returns accession json by filter
     *
     * @param params - similarity search params {@link org.genesys.server.api.v1.AccessionController.SimilaritySearchParams}
     * @return json with minimal accession data
     */
    @PostMapping(value = "/find-similar", produces = MediaType.APPLICATION_JSON_VALUE)
    public List<DuplicateFinder.SimilarityHit<Accession>> findSimilar(@RequestBody org.genesys.server.api.v1.AccessionController.SimilaritySearchParams params) throws Exception {
        List<DuplicateFinder.SimilarityHit<Accession>> results = new ArrayList<>();

        final long countFiltered = accessionApiService.countAccessions(params.select);

        if (countFiltered > 100) {
            throw new InvalidApiUsageException("Too many matches for similarity search!");
        }

        accessionProcessor.process(params.select, (accessions) -> {
            results.addAll(duplicateFinder.findSimilar(accessions, params.target));
        });

        return results;
    }

    @Accessors(fluent = true, chain = true)
    @Getter
    @Setter
    public static class SimilaritySearchParams {
        public AccessionFilter select; // Which accessions to process
        public AccessionFilter target; // What target filter to apply
    }

    /**
     * Get term overview for filters
     *
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws Exception
     */
    @PostMapping(value = "/tileIndex3min", produces = { MediaType.APPLICATION_JSON_VALUE })
    public Set<Integer> allTileIndex3min(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter) throws Exception {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        return accessionApiService.listTileIndex3min(filterInfo.filter);
    }

    /**
     * Get term overview for filters
     *
     * @param filterCode short filter code
     * @param filter the filter
     * @return the page
     * @throws Exception
     */
    @PostMapping(value = "/tileIndex3min", produces = { MediaType.APPLICATION_JSON_VALUE }, params = { "crop" })
    public Map<String, Set<Integer>> allTileIndex3minByCrop(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final AccessionFilter filter) throws Exception {

        ShortFilterService.FilterInfo<AccessionFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, AccessionFilter.class);
        return accessionApiService.listTileIndex3minByCrop(filterInfo.filter);
    }

    @GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/observations", produces = MediaType.APPLICATION_JSON_VALUE)
    public AmphibianService.AccessionObservations getAccessionObservations(@PathVariable("uuid") final UUID uuid) throws Exception {
        return amphibianService.getAccessionObservations(uuid);
    }
}