DatasetController.java

/*
 * Copyright 2024 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.api.v2.impl;

import com.google.common.collect.Sets;

import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;

import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import org.genesys.amphibian.client.model.HeatMap;
import org.genesys.amphibian.client.model.ObservationChart;
import org.genesys.amphibian.client.model.ObservationHistogram;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.FilteredPage;
import org.genesys.server.api.v2.FilteredCRUDController;
import org.genesys.server.api.v2.TranslatedCRUDController;
import org.genesys.server.api.v2.facade.DatasetApiService;
import org.genesys.server.api.v2.facade.DatasetLangApiService;
import org.genesys.server.api.v2.facade.DescriptorApiService;
import org.genesys.server.api.v2.model.dataset.DatasetAccessionRefDTO;
import org.genesys.server.api.v2.model.dataset.DatasetCreatorDTO;
import org.genesys.server.api.v2.model.dataset.DatasetDTO;
import org.genesys.server.api.v2.model.dataset.DatasetInfo;
import org.genesys.server.api.v2.model.dataset.DatasetLangDTO;
import org.genesys.server.api.v2.model.dataset.DatasetLocationDTO;
import org.genesys.server.api.v2.model.dataset.TranslatedDatasetDTO;
import org.genesys.server.api.v2.model.dataset.TranslatedDatasetInfo;
import org.genesys.server.api.v2.model.impl.DescriptorDTO;
import org.genesys.server.api.Pagination;
import org.genesys.server.api.v2.model.impl.RepositoryFileDTO;
import org.genesys.server.api.v2.model.impl.TranslatedDescriptorDTO;
import org.genesys.server.component.aspect.DownloadEndpoint;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.PublishState;
import org.genesys.server.model.dataset.Dataset;
import org.genesys.server.model.dataset.DatasetAccessionRef;
import org.genesys.server.model.dataset.DatasetLang;
import org.genesys.server.model.dataset.QDataset;
import org.genesys.server.model.filters.DatasetFilter;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.AmphibianService;
import org.genesys.server.service.DatasetService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.ShortFilterService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.service.filter.DatasetLangFilter;
import org.genesys.server.service.worker.ShortFilterProcessor;
import org.genesys.spring.CSVMessageConverter;
import org.springdoc.api.annotations.ParameterObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;

import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import javax.validation.constraints.NotNull;

@RestController("datasetApi2")
@PreAuthorize("isAuthenticated()")
@RequestMapping(DatasetController.CONTROLLER_URL)
@Tag(name = "Dataset")
public class DatasetController extends TranslatedCRUDController<DatasetDTO, TranslatedDatasetDTO,
	DatasetLangDTO, Dataset, DatasetLang, DatasetApiService, DatasetFilter> {

	/** The Constant CONTROLLER_URL. */
	public static final String CONTROLLER_URL = ApiBaseController.APIv2_BASE + "/dataset";

	public static final String CREATOR_URL = "/{uuid:\\w{8}\\-\\w{4}.+}/datasetcreator";
	public static final String FILES_URL = "/{uuid:\\w{8}\\-\\w{4}.+}/files";
	protected static final String LOCATION_URL = "/{uuid:\\w{8}\\-\\w{4}.+}/location";

	private final Set<String> terms = Sets.newHashSet("owner.uuid", "crops", "rights");

	/** The dataset service. */
	@Autowired
	protected DatasetService datasetService;

	@Autowired
	@Lazy
	private AmphibianService amphibianService;

	@Autowired
	protected ShortFilterProcessor shortFilterProcessor;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired
	private AccessionService accessionService;

	@Autowired
	protected DescriptorApiService descriptorApiService;

	@RestController("datasetLangApi2")
	@RequestMapping(DatasetController.DatasetLangController.API_URL)
	@PreAuthorize("isAuthenticated()")
	@Tag(name = "Dataset")
	public static class DatasetLangController extends FilteredCRUDController<DatasetLangDTO, DatasetLang, DatasetLangApiService, DatasetLangFilter> {
		/** The Constant API_URL. */
		public static final String API_URL = DatasetController.CONTROLLER_URL + "/lang";
	}

	@PutMapping(value = "/force-update")
	@ApiOperation(value = "Update published Dataset", notes = "Allows admins to make curatorial changes")
	public DatasetDTO forceUpdate(@RequestBody @Valid @NotNull DatasetDTO dataset) {
		return translatedApiService.forceUpdate(dataset);
	}

	@GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/descriptors")
	public List<TranslatedDescriptorDTO> getDescriptors(@PathVariable("uuid") final UUID uuid) {
		return translatedApiService.loadDescriptors(uuid);
	}

	@PostMapping(path = { "/data" })
	@ApiOperation(value = "Read accessions observations from Datatables", notes = "")
	public Page<?> filterAccessionsDatasetsData(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Descriptor UUIDs", required = true) @RequestParam List<UUID> fields, // Fields UUIDs
		final Pagination page,
		@ApiParam(value = "MCPD fields, Data filters and fields") @RequestBody AmphibianService.AccessionsDatasetsDataRequest request
	) throws Exception {

		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		var pagination = page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE);
		return amphibianService.getAccessionDatasetObservations(datasets, fields, request, pagination);
	}

	@Override
	@PutMapping(value = "", produces = { MediaType.APPLICATION_JSON_VALUE })
	public DatasetDTO update(@RequestBody @Valid @NotNull final DatasetDTO entity) {
		LOG.debug("Update Dataset {}", entity);
		return super.update(entity);
	}

	/**
	 * Gets the dataset.
	 *
	 * @param uuid the uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}")
	public TranslatedDatasetDTO getDataset(@PathVariable("uuid") final UUID uuid) throws NotFoundElement {
		LOG.debug("Load Dataset by uuid {}", uuid);
		return translatedApiService.loadTranslatedDataset(uuid);
	}

	/**
	 * Delete dataset.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @return the dataset
	 * @throws Exception the exception
	 */
	@DeleteMapping("/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO deleteDataset(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version) throws Exception {
		final DatasetDTO dataset = translatedApiService.remove(translatedApiService.getDataset(uuid, version));
		dataset.setId(null);
		return dataset;
	}

	/**
	 * Adds the descriptors.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param descriptorUuids the descriptor uuids
	 * @return the dataset
	 */
	@PostMapping(value = "/add-descriptors/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO addDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final Set<UUID> descriptorUuids) {
		return translatedApiService.addDescriptors(uuid, version, descriptorUuids);
	}

	/**
	 * Synchronize dataset descriptors with amphibian.
	 *
	 * @param uuid the dataset uuid
	 * @return the list of dataset's descriptors
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/descriptors/synchronize")
	public List<DescriptorDTO> synchronizeDescriptors(@PathVariable("uuid") final UUID uuid) throws NotFoundElement {
		return translatedApiService.synchronizeDescriptors(uuid);
	}

	/**
	 * Upsert accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param accessionRefs the accession identifiers
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/add-accessions/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO addAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
		@RequestBody final Set<DatasetAccessionRef> accessionRefs) throws NotFoundElement {

		return translatedApiService.addAccessionRefs(uuid, version, accessionRefs);
	}

	/**
	 * Set accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param accessionRefs the accession identifiers
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/set-accessions/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO setAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
		@RequestBody final Set<DatasetAccessionRef> accessionRefs) throws NotFoundElement {
		return translatedApiService.setAccessionRefs(uuid, version, accessionRefs);
	}

	/**
	 * Set accessions to Dataset from uploaded CSV file.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param separator the delimiter to use for separating entries in the CSV file
	 * @param quotechar the character to use for quoted elements in the CSV file
	 * @param file the CSV file with accessionRefs to be added
	 * @return updated record
	 */
	@PostMapping(value = "/upload-accessions/{uuid:\\w{8}\\-\\w{4}.+},{version}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public DatasetDTO uploadAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
		// CSV settings
		@RequestParam(required = false, defaultValue = "\t") char separator, @RequestParam(required = false, defaultValue = "\"") char quotechar,
		// The file
		@RequestPart(name = "file") final MultipartFile file) throws IOException {

		return translatedApiService.uploadAccessions(uuid, version, separator, quotechar, file);
	}

	/**
	 * Rematch accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/rematch-accessions/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO rematchDatasetAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version) throws NotFoundElement {
		return translatedApiService.rematchDatasetAccessions(uuid, version);
	}

	/**
	 * My datasets.
	 *
	 * @param page the page
	 * @param filterCode short filter code -- overrides filter in body
	 * @param filter the filter
	 * @return the page
	 * @throws IOException
	 */
	@PostMapping(value = "/list-mine")
	public FilteredPage<DatasetInfo, DatasetFilter> myDatasets(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
		@RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		ShortFilterService.FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
//		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetService.listDatasetsForCurrentUser(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.DESC, "lastModifiedDate")));
		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, translatedApiService.listDatasetsForCurrentUser(filterInfo.filter, page.toPageRequest(50, 20, Sort.Direction.DESC, "lastModifiedDate")));
	}

	/**
	 * List datasets.
	 *
	 * @param page the page
	 * @param filter the dataset filter
	 * @return the page
	 */
	@PostMapping(value = FilteredCRUDController.ENDPOINT_LIST, produces = { MediaType.APPLICATION_JSON_VALUE })
	public FilteredPage<TranslatedDatasetInfo, DatasetFilter> listDatasets(@ParameterObject final Pagination page, @RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		var cleanFilter = normalizeFilter(filter, DatasetFilter.class);
		return new FilteredPage<>(cleanFilter, translatedApiService.listInfo(cleanFilter, page.toPageRequest(200, 50, Sort.Direction.DESC, "firstPublishedDate")));
	}

	@Override
	@PostMapping(value = "/default-list-endpoint-disabled")
	@Operation(hidden = true)
	public FilteredPage<TranslatedDatasetDTO, DatasetFilter> list(Pagination page, DatasetFilter filter) throws SearchException, IOException {
		throw new InvalidApiUsageException("Not enabled!");
	}

	/**
	 * List datasets with suggestions.
	 *
	 * @param page the page
	 * @param filter the dataset filter
	 * @return the page with suggestions
	 */
	@PostMapping(value = FilteredCRUDController.ENDPOINT_FILTER, produces = { MediaType.APPLICATION_JSON_VALUE })
	public DatasetApiService.DatasetSuggestionPage filterDatasets(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
		@RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		ShortFilterService.FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);

		var pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, translatedApiService.listInfo(filterInfo.filter, page.toPageRequest(200, 50, Sort.Direction.DESC, "firstPublishedDate")));
		Map<String, ElasticsearchService.TermResult> suggestionRes = translatedApiService.getSuggestions(filterInfo.filter);

		return DatasetApiService.DatasetSuggestionPage.from(pageRes, suggestionRes);
	}

	@Override
	@PostMapping(value = "/default-filter-endpoint-disabled")
	@Operation(hidden = true)
	public FilteredPage<TranslatedDatasetDTO, DatasetFilter> filter(String filterCode, Pagination page, DatasetFilter filter) throws IOException, SearchException {
			throw new InvalidApiUsageException("Not enabled!");
	}

	/**
	 * Get term overview for filters
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the overview
	 * @throws SearchException
	 */
	@PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
	public DatasetService.DatasetOverview overview(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final DatasetFilter filter,
		@RequestParam(name = "limit", defaultValue = "10", required = false) final int limit)
		throws IOException, SearchException, ExecutionException, InterruptedException {

		ShortFilterService.FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
		filterInfo.filter
			.state(Sets.newHashSet(PublishState.PUBLISHED))
			.current(true);

		Map<String, ElasticsearchService.TermResult> overview = elasticsearchService.termStatisticsAuto(org.genesys.server.model.dataset.Dataset.class, filterInfo.filter, Math.min(50, limit), terms.toArray(new String[] {}));

		ElasticsearchService.TermResult crops = elasticsearchService.recountResult(org.genesys.server.model.dataset.Dataset.class, QDataset.dataset.crops, filterInfo.filter, overview.get("crops"), "crops");
		overview.put("crops", crops);

		long datasetCount = translatedApiService.countDatasets(filterInfo.filter);
		Map<String, ElasticsearchService.TermResult> suggestionRes = translatedApiService.getSuggestions(filterInfo.filter);

		return DatasetService.DatasetOverview.from(filterInfo.filterCode, filterInfo.filter, overview, datasetCount, suggestionRes);
	}

	/**
	 * Load more data for the specified term
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param term the term
	 * @return the term result
	 * @throws SearchException the search exception
	 * @throws IOException signals that an I/O exception has occurred
	 */
	@PostMapping(value = "/overview/{term}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public ElasticsearchService.TermResult loadMoreTerms(@PathVariable(name = "term") final String term, @RequestBody(required = false) final DatasetFilter filter,
		@RequestParam(name = "f", required = false) final String filterCode, @RequestParam(name = "limit", defaultValue = "20", required = false) final int limit)
		throws IOException, SearchException, ExecutionException, InterruptedException {

		if (!terms.contains(term)) {
			throw new InvalidApiUsageException("No such term. Will not search.");
		}

		ShortFilterService.FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
		filterInfo.filter
			.state(Sets.newHashSet(PublishState.PUBLISHED))
			.current(true);

		ElasticsearchService.TermResult termResult = elasticsearchService.termStatisticsAuto(org.genesys.server.model.dataset.Dataset.class, filterInfo.filter, Math.min(200, limit), term);
		if (term.equals("crops")) {
			termResult = elasticsearchService.recountResult(org.genesys.server.model.dataset.Dataset.class, QDataset.dataset.crops, filterInfo.filter, termResult, "crops");
		}
		return termResult;
	}

	/**
	 * Load AccessionRef list by Dataset
	 *
	 * @param uuid uuid of Dataset
	 * @param page Pageable
	 * @return the page
	 * @throws NotFoundElement
	 */
	@GetMapping(value = "/accessions/{uuid:\\w{8}\\-\\w{4}.+}", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
	public Page<DatasetAccessionRefDTO> listAccessions(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) throws NotFoundElement {
		return translatedApiService.listAccessions(uuid, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE));
	}

	/**
	 * Load full accessions list by Dataset
	 *
	 * @param uuid uuid of Subset
	 * @param page Pageable
	 * @return the page
	 * @throws NotFoundElement
	 */
	@GetMapping(value = "/accessions/{uuid:\\w{8}\\-\\w{4}.+}", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE }, params = { "full" })
	public Page<Accession> listFullAccessions(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) throws NotFoundElement, SearchException {
		AccessionFilter filter = new AccessionFilter()
			.datasets(Sets.newHashSet(uuid));
		return accessionService.list(filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE));
	}

	/**
	 * Removes the descriptors.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param descriptorUuids the descriptor uuids
	 * @return the dataset
	 */
	@PostMapping(value = "/remove-descriptors/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO removeDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final Set<UUID> descriptorUuids) {
		return translatedApiService.removeDescriptors(uuid, version, descriptorUuids);
	}

	@PostMapping(value = "/set-descriptors/{uuid:\\w{8}\\-\\w{4}.+},{version}")
	public DatasetDTO updateDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final List<UUID> descriptorUuids) {
		return translatedApiService.updateDescriptors(uuid, version, descriptorUuids);
	}

	/**
	 * Loads dataset by uuid and version and tries to publish it.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return published Dataset (admin-only)
	 */
	@RequestMapping(value = "/approve", method = RequestMethod.POST)
	public DatasetDTO approveDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final DatasetDTO dataset = translatedApiService.getDataset(uuid, version);
		return translatedApiService.approveDataset(dataset);
	}

	/**
	 * Loads dataset by uuid and version and send to review.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return dataset in review state
	 */
	@RequestMapping(value = "/for-review", method = RequestMethod.POST)
	public DatasetDTO reviewDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final DatasetDTO dataset = translatedApiService.getDataset(uuid, version);
		return translatedApiService.reviewDataset(dataset);
	}

	/**
	 * Loads dataset by uuid and version and unpublish it.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return unpublished dataset
	 */
	@RequestMapping(value = "/reject", method = RequestMethod.POST)
	public DatasetDTO rejectDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final DatasetDTO dataset = translatedApiService.getDataset(uuid, version);
		return translatedApiService.rejectDataset(dataset);
	}

	/**
	 * Autocomplete.
	 *
	 * @param text the text
	 * @return the list
	 */
	@GetMapping(value = CREATOR_URL + "/autocomplete", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<DatasetCreatorDTO> autocomplete(@RequestParam("c") final String text) {
		if (text.length() < 3) {
			return Collections.emptyList();
		}

		return translatedApiService.autocompleteCreators(text);
	}

	/**
	 * Creates the dataset creator.
	 *
	 * @param uuid the uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = CREATOR_URL + "/create")
	public DatasetCreatorDTO createDatasetCreator(@PathVariable("uuid") final UUID uuid, @RequestBody final DatasetCreatorDTO datasetCreator) throws NotFoundElement {

		return translatedApiService.createDatasetCreator(uuid, datasetCreator);
	}

	/**
	 * Delete dataset creator.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	// uses request body
	@RequestMapping(value = CREATOR_URL + "/delete", method = { RequestMethod.POST, RequestMethod.DELETE })
	public DatasetCreatorDTO deleteDatasetCreator(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetCreatorDTO datasetCreator) throws NotFoundElement {
		return translatedApiService.removeDatasetCreator(datasetUuid, datasetCreator);
	}

	/**
	 * List dataset creators.
	 *
	 * @param page the page
	 * @param uuid the uuid
	 * @return the page
	 */
	@GetMapping(value = CREATOR_URL + "/list")
	public Page<DatasetCreatorDTO> listDatasetCreators(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) {
		return translatedApiService.listDatasetCreators(uuid, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id"));
	}

	/**
	 * Update dataset creator.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = CREATOR_URL + "/update")
	public DatasetCreatorDTO updateDatasetCreator(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetCreatorDTO datasetCreator) throws NotFoundElement {
		return translatedApiService.updateDatasetCreator(datasetUuid, datasetCreator);
	}

	/**
	 * Load by uuid.
	 *
	 * @param uuid the uuid
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = CREATOR_URL + "/{creatorUuid}")
	public DatasetCreatorDTO loadByUuid(@PathVariable("uuid") final UUID uuid, @PathVariable("creatorUuid") final UUID creatorUuid) throws NotFoundElement {
		return translatedApiService.loadDatasetCreator(creatorUuid);
	}

	/**
	 * Adds the file to dataset.
	 *
	 * @param inputFile the input file
	 * @param datasetUuid the dataset uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 */
	@PostMapping(value = FILES_URL + "/add")
	public DatasetDTO addFileToDataset(@RequestParam("file") final MultipartFile inputFile, @PathVariable("uuid") final UUID datasetUuid) throws NotFoundElement,
		InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
		LOG.info("Upload file to dataset by uuid {}", datasetUuid);
		return translatedApiService.addDatasetFile(translatedApiService.getDataset(datasetUuid, null), inputFile);
	}

	/**
	 * Removes the file of dataset.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param fileUuid the file uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 * @throws NoSuchRepositoryFileException the no such repository file exception
	 */
	@DeleteMapping(value = FILES_URL + "/delete/{fileUuid}")
	public DatasetDTO removeFileOfDataset(@PathVariable("uuid") final UUID datasetUuid, @PathVariable("fileUuid") final UUID fileUuid) throws NotFoundElement,
		InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException, NoSuchRepositoryFileException {
		return translatedApiService.removeDatasetFile(translatedApiService.getDataset(datasetUuid, null), fileUuid);
	}

	/**
	 * Gets the list.
	 *
	 * @param datasetUuid the dataset uuid
	 * @return the list
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = FILES_URL + "/list")
	public List<RepositoryFileDTO> getList(@PathVariable("uuid") final UUID datasetUuid) throws NotFoundElement {
		return translatedApiService.listDatasetFiles(translatedApiService.loadDataset(datasetUuid));
	}

	/**
	 * Update dataset file.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param metadata the metadata
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 * @throws NoSuchRepositoryFileException the no such repository file exception
	 */
	@PostMapping(value = FILES_URL + "/update")
	public DatasetDTO updateDatasetFile(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final RepositoryFileDTO metadata) throws NotFoundElement, NoSuchRepositoryFileException {
		return translatedApiService.updateDatasetFile(translatedApiService.getDataset(datasetUuid, null), metadata);
	}

	/**
	 * Creates the location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = LOCATION_URL + "/create")
	public DatasetLocationDTO createLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocationDTO datasetLocation) throws NotFoundElement {
		return translatedApiService.createLocation(datasetUuid, datasetLocation);
	}

	/**
	 * Delete location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@RequestMapping(value = LOCATION_URL + "/delete", method = { RequestMethod.POST, RequestMethod.DELETE })
	public DatasetLocationDTO deleteLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocationDTO datasetLocation) throws NotFoundElement {
		return translatedApiService.removeLocation(datasetUuid, datasetLocation);
	}

	/**
	 * List location.
	 *
	 * @param page the page
	 * @param uuid the uuid
	 * @return the page
	 */
	@GetMapping(value = LOCATION_URL + "/list")
	public Page<DatasetLocationDTO> listLocation(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) {
		return translatedApiService.listLocation(uuid, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id"));
	}

	/**
	 * Update location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = LOCATION_URL + "/update")
	public DatasetLocationDTO updateLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocationDTO datasetLocation) throws NotFoundElement {
		return translatedApiService.updateLocation(datasetUuid, datasetLocation);
	}

	/**
	 * Load by uuid.
	 *
	 * @param locationUuid the location uuid
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = LOCATION_URL + "/{locationUuid}")
	public DatasetLocationDTO loadLocationByUuid(@PathVariable("locationUuid") final UUID locationUuid) throws NotFoundElement {
		return translatedApiService.loadLocation(locationUuid);
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/download", method = RequestMethod.POST, params = { "mcpd" })
	public void downloadMcpd(@PathVariable("uuid") final UUID datasetUuid, HttpServletResponse response) throws IOException, NotFoundElement {
		final DatasetDTO dataset = translatedApiService.getDataset(datasetUuid, null);
		if (dataset == null) {
			throw new NotFoundElement();
		}

		// Write MCPD to the stream.
		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"MCPD - %1s.xlsx\"", dataset.getUuid()));
		// response.flushBuffer();

		final OutputStream outputStream = response.getOutputStream();
		try {
			translatedApiService.writeXlsxMCPD(dataset, outputStream);
			response.flushBuffer();
		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		}
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/download", method = RequestMethod.POST, params = { "metadata" })
	public void downloadMetadata(@PathVariable("uuid") final UUID datasetUuid, final HttpServletResponse response)
		throws IOException, NoSuchRepositoryFileException {

		translatedApiService.downloadMetadata(datasetUuid, response);
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid:\\w{8}\\-\\w{4}.+}/download/{fileId}", method = RequestMethod.POST)
	public void downloadDatasetFile(@PathVariable("uuid") final UUID datasetUuid, @PathVariable("fileId") final long fileId, final HttpServletResponse response)
		throws IOException, NoSuchRepositoryFileException {

		translatedApiService.downloadDatasetFile(datasetUuid, fileId, response);
	}

	/**
	 * Create a new version of Dataset based on an existing published Dataset.
	 *
	 * @param uuid Dataset UUID
	 * @return the new version of Dataset
	 */
	@PostMapping(value = "/create-new-version")
	public DatasetDTO createNewVersion(@RequestParam(value = "uuid", required = true) final UUID uuid) {
		final DatasetDTO dataset = translatedApiService.loadDataset(uuid);
		return translatedApiService.createNewVersion(dataset);
	}

	@PostMapping(value = "/accessions-datasets")
	public Set<UUID> findDatasetsByAccessions(@RequestBody(required = false) AccessionFilter filter) {
		return translatedApiService.findAccessionsAmphibianDatasets(filter);
	}

	@PostMapping(path = { "/data/charts" })
	@ApiOperation(value = "getAccessionDatasetCharts", notes = "Read accession observation charts from Datatables")
	public List<ObservationChart> getAccessionDatasetCharts(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Descriptor UUIDs", required = true) @RequestParam List<UUID> fields, // Fields UUIDs
		@ApiParam(value = "Data filters and fields") @RequestBody(required = true) AmphibianService.TraitFilters filter
	) throws SearchException {

		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		return amphibianService.getDatasetsCharts(datasets, fields, filter);
	}

	@PostMapping(path = { "/data/heatmap" })
	@ApiOperation(value = "getAccessionsObservationsHeatMap", notes = "Calculate a heat map of observations for two selected categorical descriptors in Datasets")
	public HeatMap getAccessionsObservationsHeatMap(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "X category field uuid") @RequestParam(name = "xCategory") UUID xCategoryField, // x category field uuid
		@ApiParam(value = "Y category field uuid") @RequestParam(name = "yCategory") UUID yCategoryField, // y category field uuid
		@ApiParam(value = "Data filters", required = true) @RequestBody(required = false) AmphibianService.TraitFilters filter // data  filters
	) throws SearchException {
		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		var xCategoryDescriptor = descriptorApiService.getDescriptor(xCategoryField);
		var yCategoryDescriptor = descriptorApiService.getDescriptor(yCategoryField);
		if (xCategoryDescriptor == null || yCategoryDescriptor == null) {
			throw new NotFoundElement("No such categorical descriptor");
		}
		if (!xCategoryDescriptor.getDataType().equals(org.genesys.server.model.traits.Descriptor.DataType.CODED)
			|| !yCategoryDescriptor.getDataType().equals(org.genesys.server.model.traits.Descriptor.DataType.CODED)) {

			throw new InvalidApiUsageException("Categorical Descriptor must have CODED data type");
		}
		return amphibianService.getDatasetsHeatMap(datasets, xCategoryDescriptor.getUuid(), yCategoryDescriptor.getUuid(), filter);
	}

	@PostMapping(path = { "/data/histogram" })
	@ApiOperation(value = "getObservationsHistograms", notes = "Get histogram of numerical observations from datasets")
	public List<ObservationHistogram> getObservationHistograms(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Specify fields to return") @RequestParam(name = "fields") List<UUID> fields, // selected fields
		@ApiParam(value = "Number of bins") @RequestParam(name = "bins", required = false, defaultValue = "4") Integer binsNumber, // number of bins
		@ApiParam(value = "Data filters", required = true) @RequestBody(required = false) AmphibianService.TraitFilters filter // data  filters
	) throws SearchException {
		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		return amphibianService.getDatasetsHistograms(datasets, fields, filter, binsNumber);
	}


	@PostMapping(value = "/{datasetUuid:\\w{8}\\-\\w{4}.+}/cluster")
	public void clusterDataset(
		@ApiParam(value = "Dataset UUID", required = true) @PathVariable UUID datasetUuid, // Dataset UUID
		@ApiParam(value = "Data filters and fields") @RequestBody(required = true) DatasetApiService.ClusterRequest dataRequest,
		HttpServletResponse response
	) throws Exception {

		translatedApiService.clusterDataset(() -> {
			try {
				// Proxy to client
				response.setBufferSize(4*1024);
				response.setContentType("application/json");
				return response.getOutputStream();
			} catch (IOException e) {
				return null;
			}
		}, datasetUuid, dataRequest.fields, dataRequest.filters, dataRequest.options);
	}

	@PostMapping(value = "/{datasetUuid:\\w{8}\\-\\w{4}.+}/core")
	public void coreDataset(
		@ApiParam(value = "Dataset UUID", required = true) @PathVariable UUID datasetUuid, // Dataset UUID
		@ApiParam(value = "Data filters and fields") @RequestBody(required = true) DatasetApiService.ClusterRequest dataRequest,
		HttpServletResponse response
	) throws Exception {

		translatedApiService.coreCollection(() -> {
			try {
				// Proxy to client
				response.setBufferSize(4*1024);
				response.setContentType("application/json");
				return response.getOutputStream();
			} catch (IOException e) {
				return null;
			}
		}, datasetUuid, dataRequest.fields, dataRequest.filters, dataRequest.options);
	}

}