DatasetController.java

/*
 * Copyright 2018 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.api.v1;

import static org.genesys.server.service.DatasetService.DATASET_METADATA_FILE_NAME;

import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;

import javax.servlet.http.HttpServletResponse;
import javax.validation.ConstraintViolation;
import javax.validation.Validator;

import org.genesys.amphibian.client.model.HeatMap;
import org.genesys.amphibian.client.model.ObservationChart;
import org.genesys.amphibian.client.model.ObservationHistogram;
import org.genesys.blocks.model.JsonViews;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.FilteredPage;
import org.genesys.server.api.Pagination;
import org.genesys.server.api.v1.facade.DatasetApiService;
import org.genesys.server.api.v1.facade.DescriptorApiService;
import org.genesys.server.api.v1.mapper.APIv1Mapper;
import org.genesys.server.api.v1.model.Dataset;
import org.genesys.server.api.v1.model.Descriptor;
import org.genesys.server.component.aspect.DownloadEndpoint;
import org.genesys.server.exception.DetailedConstraintViolationException;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.model.PublishState;
import org.genesys.server.model.dataset.DatasetAccessionRef;
import org.genesys.server.model.dataset.DatasetCreator;
import org.genesys.server.model.dataset.DatasetLocation;
import org.genesys.server.model.dataset.QDataset;
import org.genesys.server.model.filters.DatasetFilter;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.AmphibianService;
import org.genesys.server.service.DatasetService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.PartnerService;
import org.genesys.server.service.AmphibianService.AccessionsDatasetsDataRequest;
import org.genesys.server.service.AmphibianService.TraitFilters;
import org.genesys.server.service.DatasetService.DatasetOverview;
import org.genesys.server.service.ElasticsearchService.TermResult;
import org.genesys.server.service.ShortFilterService.FilterInfo;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.exception.SearchException;
import org.genesys.server.service.worker.ShortFilterProcessor;
import org.genesys.spring.CSVMessageConverter;
import org.genesys.taxonomy.gringlobal.component.CabReader;
import org.springdoc.api.annotations.ParameterObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import com.fasterxml.jackson.annotation.JsonView;
import com.google.common.collect.Sets;
import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;

import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;

/**
 * The Class DatasetController.
 *
 * @author Andrey Lugovskoy
 * @author Matija Obreza
 */
@RestController("datasetApi1")
@RequestMapping(org.genesys.server.api.v1.DatasetController.CONTROLLER_URL)
@PreAuthorize("isAuthenticated()")
@Api(tags = { "dataset" })
public class DatasetController extends ApiBaseController {

	// Rest controller base URL
	public static final String CONTROLLER_URL = ApiBaseController.APIv1_BASE + "/dataset";

	public static final String CREATOR_URL = "/{uuid}/datasetcreator";
	public static final String FILES_URL = "/{uuid}/files";
	protected static final String LOCATION_URL = "/{uuid}/location";

	private final Set<String> terms = Sets.newHashSet("owner.uuid", "crops", "rights");

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	/** The dataset service. */
	@Autowired
	protected DatasetApiService datasetApiService;

	@Autowired
	protected DatasetService datasetService;
	
	@Autowired
	private APIv1Mapper mapper;

	/** The short filter service. */
	@Autowired
	protected ShortFilterProcessor shortFilterProcessor;

	/** The descriptor service. */
	@Autowired
	protected DescriptorApiService descriptorApiService;

	/** The accession service. */
	@Autowired
	private AccessionService accessionService;

	@Autowired
	private Validator validator;

	@Autowired
	private RepositoryService repositoryService;

	@Autowired
	private PartnerService partnerService;

	@Autowired
	@Lazy
	private AmphibianService amphibianService;

	/**
	 * Creates the dataset.
	 *
	 * @param dataset the dataset
	 * @return the dataset
	 */
	@PostMapping(value = "/create")
	@PreAuthorize("isAuthenticated()")
	public Dataset createDataset(@RequestBody final Dataset dataset) {
		LOG.info("Create Dataset {}", dataset);
		if (dataset.getOwner() == null || dataset.getOwner().getUuid() == null) {
			throw new InvalidApiUsageException("owner.uuid not provided");
		}
		dataset.setOwner(mapper.mapInfo(partnerService.get(dataset.getOwner().getUuid())));
		return datasetApiService.create(dataset);
	}

	/**
	 * Update dataset.
	 *
	 * @param dataset the dataset
	 * @return the dataset
	 */
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#dataset, 'write')")
	@PostMapping(value = "/update")
	public Dataset updateDataset(@RequestBody final Dataset dataset) {
		LOG.debug("Update Dataset {}", dataset);
		dataset.setOwner(mapper.mapInfo(partnerService.get(dataset.getOwner().getUuid())));
		return datasetApiService.update(dataset);
	}

	/**
	 * Gets the dataset.
	 *
	 * @param uuid the uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = "/{uuid}")
	public Dataset getDataset(@PathVariable("uuid") final UUID uuid) throws NotFoundElement {
		LOG.debug("Load Dataset by uuid {}", uuid);
		return datasetApiService.loadDataset(uuid);
	}

	/**
	 * Delete dataset.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @return the dataset
	 * @throws Exception the exception
	 */
	@DeleteMapping("/{uuid},{version}")
	public Dataset deleteDataset(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version) throws Exception {
		final Dataset dataset = datasetApiService.remove(datasetApiService.getDataset(uuid, version));
		dataset.setId(null);
		return dataset;
	}

	/**
	 * Adds the descriptors.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param descriptorUuids the descriptor uuids
	 * @return the dataset
	 */
	@PostMapping(value = "/add-descriptors/{uuid},{version}")
	public Dataset addDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final Set<UUID> descriptorUuids) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		LOG.debug("Got dataset {}", dataset);
		final Set<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorApiService.getDescriptor(descriptorUuid)).collect(Collectors.toSet());
		LOG.debug("Got {} descriptors", descriptors.size());
		return datasetApiService.addDescriptors(dataset, descriptors.toArray(new Descriptor[] {}));
	}

	/**
	 * Gets dataset descriptors.
	 *
	 * @param uuid the dataset uuid
	 * @return the list of loaded descriptors
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = "/{uuid}/descriptors")
	public List<Descriptor> getDescriptors(@PathVariable("uuid") final UUID uuid) throws NotFoundElement {
		LOG.debug("Load Dataset by uuid {}", uuid);
		var dataset = datasetApiService.loadDataset(uuid);
		LOG.debug("Got dataset {}", dataset);
		return datasetApiService.getDatasetDescriptors(dataset);
	}

	/**
	 * Synchronize dataset descriptors with amphibian.
	 *
	 * @param uuid the dataset uuid
	 * @return the list of dataset's descriptors
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/{uuid}/descriptors/synchronize")
	public List<Descriptor> synchronizeDescriptors(@PathVariable("uuid") final UUID uuid) throws NotFoundElement {
		LOG.debug("Load Dataset by uuid {}", uuid);
		var dataset = datasetApiService.loadDataset(uuid);
		return datasetApiService.synchronizeDescriptors(dataset);
	}

	/**
	 * Upsert accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param accessionRefs the accession identifiers
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/add-accessions/{uuid},{version}")
	public Dataset addAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
			@RequestBody final Set<DatasetAccessionRef> accessionRefs) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		LOG.info("Want to add {} accessionRefs to dataset {}", accessionRefs.size(), dataset.getUuid());

		return datasetApiService.addAccessionRefs(dataset, accessionRefs);
	}

	/**
	 * Set accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param accessionRefs the accession identifiers
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/set-accessions/{uuid},{version}")
	public Dataset setAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
			@RequestBody final Set<DatasetAccessionRef> accessionRefs) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		return datasetApiService.setAccessionRefs(dataset, accessionRefs);
	}

	/**
	 * Set accessions to Dataset from uploaded CSV file.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param separator the delimiter to use for separating entries in the CSV file
	 * @param quotechar the character to use for quoted elements in the CSV file
	 * @param file the CSV file with accessionRefs to be added
	 * @return updated record
	 */
	@PostMapping(value = "/upload-accessions/{uuid},{version}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public Dataset uploadAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version,
			// CSV settings
			@RequestParam(required = false, defaultValue = "\t") char separator, @RequestParam(required = false, defaultValue = "\"") char quotechar,
			// The file
			@RequestPart(name = "file") final MultipartFile file) throws IOException {

		// Permit only a CSV file
		if (!file.getContentType().equalsIgnoreCase("text/csv")) {
			throw new InvalidApiUsageException("Invalid file type: " + file.getContentType() + " is not permitted.");
		}

		Dataset dataset = datasetApiService.getDataset(uuid, version);
		List<DatasetAccessionRef> accessionRefs = new ArrayList<>();

		// Build CSV parser
		CSVParser csvParser = new CSVParserBuilder().withSeparator(separator).withQuoteChar(quotechar).withEscapeChar((char) 0)
				.withStrictQuotes(false).withIgnoreLeadingWhiteSpace(false).withIgnoreQuotations(true).build();
		// Read file bytes as CSV
		try (CSVReader reader = new CSVReaderBuilder(CabReader.bomSafeReader(file.getInputStream())).withSkipLines(0).withCSVParser(csvParser).build()) {
			Iterator<DatasetAccessionRef> beanReader = CabReader.beanReader(DatasetAccessionRef.class, reader).iterator();
			DatasetAccessionRef acceRef = null;
			while (beanReader.hasNext() && (acceRef = beanReader.next()) != null) {
				Set<ConstraintViolation<DatasetAccessionRef>> violations = validator.validate(acceRef);
				if (violations == null || violations.isEmpty()) {
					accessionRefs.add(acceRef);
				} else {
					throw new DetailedConstraintViolationException("Failed to read CSV file in line " + reader.getLinesRead(), violations);
				}
			}
		}

		return datasetApiService.setAccessionRefs(dataset, accessionRefs);
	}

	/**
	 * Rematch accessions.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = "/rematch-accessions/{uuid},{version}")
	public Dataset rematchDatasetAccessions(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		return datasetApiService.rematchDatasetAccessions(dataset);
	}

	/**
	 * My datasets.
	 *
	 * @param page the page
	 * @param filterCode short filter code -- overrides filter in body
	 * @param filter the filter
	 * @return the page
	 * @throws IOException
	 */
	@PostMapping(value = "/list-mine")
	public FilteredPage<Dataset, DatasetFilter> myDatasets(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
			@RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
//		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetService.listDatasetsForCurrentUser(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.DESC, "lastModifiedDate")));
		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetApiService.listDatasetsForCurrentUser(filterInfo.filter, page.toPageRequest(50, 20, Sort.Direction.DESC, "lastModifiedDate")));
	}

	/**
	 * List datasets.
	 *
	 * @param page the page
	 * @param filter the dataset filter
	 * @return the page
	 */
	@PostMapping(value = "/list")
	public FilteredPage<Dataset, DatasetFilter> datasetList(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
			@RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
//		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id")));
		return new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetApiService.list(filterInfo.filter, page.toPageRequest(50, 20, Sort.Direction.ASC, "id")));
	}

	/**
	 * List datasets with suggestions.
	 *
	 * @param page the page
	 * @param filter the dataset filter
	 * @return the page with suggestions
	 */
	@PostMapping(value = "/filter")
	public DatasetApiService.DatasetSuggestionPage listSuggestions(@RequestParam(name = "f", required = false) String filterCode, @ParameterObject final Pagination page,
			@RequestBody(required = false) DatasetFilter filter) throws IOException, SearchException {

		FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);

//		FilteredPage<Dataset> pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetService.list(filterInfo.filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id")));
		var pageRes = new FilteredPage<>(filterInfo.filterCode, filterInfo.filter, datasetApiService.list(filterInfo.filter, page.toPageRequest(50, 20, Sort.Direction.ASC, "id")));
		Map<String, ElasticsearchService.TermResult> suggestionRes = datasetApiService.getSuggestions(filterInfo.filter);

		return DatasetApiService.DatasetSuggestionPage.from(pageRes, suggestionRes);
	}

	/**
	 * Get term overview for filters
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @return the overview
	 * @throws SearchException
	 */
	@PostMapping(value = "/overview", produces = { MediaType.APPLICATION_JSON_VALUE })
	@JsonView({ JsonViews.Public.class })
	public DatasetOverview overview(@RequestParam(name = "f", required = false) final String filterCode, @RequestBody(required = false) final DatasetFilter filter,
			@RequestParam(name = "limit", defaultValue = "10", required = false) final int limit)
			throws IOException, SearchException, ExecutionException, InterruptedException {

		FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
		filterInfo.filter
			.state(Sets.newHashSet(PublishState.PUBLISHED))
			.current(true);

		Map<String, TermResult> overview = elasticsearchService.termStatisticsAuto(org.genesys.server.model.dataset.Dataset.class, filterInfo.filter, Math.min(50, limit), terms.toArray(new String[] {}));

		TermResult crops = elasticsearchService.recountResult(org.genesys.server.model.dataset.Dataset.class, QDataset.dataset.crops, filterInfo.filter, overview.get("crops"), "crops");
		overview.put("crops", crops);

		long datasetCount = datasetApiService.countDatasets(filterInfo.filter);
		Map<String, TermResult> suggestionRes = datasetApiService.getSuggestions(filterInfo.filter);

		return DatasetOverview.from(filterInfo.filterCode, filterInfo.filter, overview, datasetCount, suggestionRes);
	}

	/**
	 * Load more data for the specified term
	 *
	 * @param filterCode short filter code
	 * @param filter the filter
	 * @param term the term
	 * @return the term result
	 * @throws SearchException the search exception
	 * @throws IOException signals that an I/O exception has occurred
	 */
	@PostMapping(value = "/overview/{term}", produces = { MediaType.APPLICATION_JSON_VALUE })
	public TermResult loadMoreTerms(@PathVariable(name = "term") final String term, @RequestBody(required = false) final DatasetFilter filter,
			@RequestParam(name = "f", required = false) final String filterCode, @RequestParam(name = "limit", defaultValue = "20", required = false) final int limit)
			throws IOException, SearchException, ExecutionException, InterruptedException {

		if (! terms.contains(term)) {
			throw new InvalidApiUsageException("No such term. Will not search.");
		}

		FilterInfo<DatasetFilter> filterInfo = shortFilterProcessor.processFilter(filterCode, filter, DatasetFilter.class);
		filterInfo.filter
			.state(Sets.newHashSet(PublishState.PUBLISHED))
			.current(true);

		TermResult termResult = elasticsearchService.termStatisticsAuto(org.genesys.server.model.dataset.Dataset.class, filterInfo.filter, Math.min(200, limit), term);
		if (term.equals("crops")) {
			termResult = elasticsearchService.recountResult(org.genesys.server.model.dataset.Dataset.class, QDataset.dataset.crops, filterInfo.filter, termResult, "crops");
		}
		return termResult;
	}

	/**
	 * Load AccessionRef list by Dataset
	 *
	 * @param uuid uuid of Dataset
	 * @param page Pageable
	 * @return the page
	 * @throws NotFoundElement
	 */
	@JsonView({ JsonViews.Public.class })
	@GetMapping(value = "/accessions/{uuid}", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE })
	public Page<DatasetAccessionRef> listAccessions(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) throws NotFoundElement {
		return datasetApiService.listAccessions(datasetApiService.getDataset(uuid, null), page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE));
	}

	/**
	 * Load full accessions list by Dataset
	 *
	 * @param uuid uuid of Subset
	 * @param page Pageable
	 * @return the page
	 * @throws NotFoundElement
	 */
	@JsonView({ JsonViews.Public.class })
	@GetMapping(value = "/accessions/{uuid}", produces = { MediaType.APPLICATION_JSON_VALUE, CSVMessageConverter.TEXT_CSV_VALUE }, params = { "full" })
	public Page<Accession> listFullAccessions(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) throws NotFoundElement, SearchException {
		AccessionFilter filter = new AccessionFilter()
			.datasets(Sets.newHashSet(uuid));
		return accessionService.list(filter, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE));
	}

	/**
	 * Removes the descriptors.
	 *
	 * @param uuid the uuid
	 * @param version the version
	 * @param descriptorUuids the descriptor uuids
	 * @return the dataset
	 */
	@PostMapping(value = "/remove-descriptors/{uuid},{version}")
	public Dataset removeDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final Set<UUID> descriptorUuids) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		final Set<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorApiService.getDescriptor(descriptorUuid)).collect(Collectors.toSet());
		return datasetApiService.removeDescriptors(dataset, descriptors.toArray(new Descriptor[] {}));
	}

	@PostMapping(value = "/set-descriptors/{uuid},{version}")
	public Dataset updateDescriptors(@PathVariable("uuid") final UUID uuid, @PathVariable("version") final int version, @RequestBody final List<UUID> descriptorUuids) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		final List<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorApiService.getDescriptor(descriptorUuid)).collect(Collectors.toList());
		return datasetApiService.updateDescriptors(dataset, descriptors);
	}

	/**
	 * Loads dataset by uuid and version and tries to publish it.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return published Dataset (admin-only)
	 */
	@RequestMapping(value = "/approve", method = RequestMethod.POST)
	public Dataset approveDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		return datasetApiService.approveDataset(dataset);
	}

	/**
	 * Loads dataset by uuid and version and send to review.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return dataset in review state
	 */
	@RequestMapping(value = "/for-review", method = RequestMethod.POST)
	public Dataset reviewDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		return datasetApiService.reviewDataset(dataset);
	}

	/**
	 * Loads dataset by uuid and version and unpublish it.
	 *
	 * @param uuid dataset UUID
	 * @param version record version
	 * @return unpublished dataset
	 */
	@RequestMapping(value = "/reject", method = RequestMethod.POST)
	public Dataset rejectDataset(@RequestParam(value = "uuid", required = true) final UUID uuid, @RequestParam(value = "version", required = true) final int version) {
		final Dataset dataset = datasetApiService.getDataset(uuid, version);
		return datasetApiService.rejectDataset(dataset);
	}

	/**
	 * Autocomplete.
	 *
	 * @param text the text
	 * @return the list
	 */
	@JsonView(JsonViews.Public.class)
	@GetMapping(value = CREATOR_URL + "/autocomplete", produces = MediaType.APPLICATION_JSON_VALUE)
	public List<DatasetCreator> autocomplete(@RequestParam("c") final String text) {
		if (text.length() < 3) {
			return Collections.emptyList();
		}

		return datasetApiService.autocompleteCreators(text);
	}

	/**
	 * Creates the dataset creator.
	 *
	 * @param uuid the uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = CREATOR_URL + "/create")
	public DatasetCreator createDatasetCreator(@PathVariable("uuid") final UUID uuid, @RequestBody final DatasetCreator datasetCreator) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(uuid, null);
		return datasetApiService.createDatasetCreator(dataset, datasetCreator);
	}

	/**
	 * Delete dataset creator.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	// uses request body
	@RequestMapping(value = CREATOR_URL + "/delete", method = { RequestMethod.POST, RequestMethod.DELETE })
	public DatasetCreator deleteDatasetCreator(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetCreator datasetCreator) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		return datasetApiService.removeDatasetCreator(dataset, datasetCreator);
	}

	/**
	 * List dataset creators.
	 *
	 * @param page the page
	 * @param uuid the uuid
	 * @return the page
	 */
	@GetMapping(value = CREATOR_URL + "/list")
	public Page<DatasetCreator> listDatasetCreators(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) {
		return datasetApiService.listDatasetCreators(uuid, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id"));
	}

	/**
	 * Update dataset creator.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetCreator the dataset creator
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = CREATOR_URL + "/update")
	public DatasetCreator updateDatasetCreator(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetCreator datasetCreator) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		return datasetApiService.updateDatasetCreator(dataset, datasetCreator);
	}

	/**
	 * Load by uuid.
	 *
	 * @param uuid the uuid
	 * @return the dataset creator
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = CREATOR_URL + "/{creatorUuid}")
	public DatasetCreator loadByUuid(@PathVariable("uuid") final UUID uuid, @PathVariable("creatorUuid") final UUID creatorUuid) throws NotFoundElement {
		return datasetApiService.loadDatasetCreator(creatorUuid);
	}

	/**
	 * Adds the file to dataset.
	 *
	 * @param inputFile the input file
	 * @param datasetUuid the dataset uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 */
	@PostMapping(value = FILES_URL +"/add")
	public Dataset addFileToDataset(@RequestParam("file") final MultipartFile inputFile, @PathVariable("uuid") final UUID datasetUuid) throws NotFoundElement,
			InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException {
		LOG.info("Upload file to dataset by uuid {}", datasetUuid);
		return datasetApiService.addDatasetFile(datasetApiService.getDataset(datasetUuid, null), inputFile);
	}

	/**
	 * Removes the file of dataset.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param fileUuid the file uuid
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 * @throws NoSuchRepositoryFileException the no such repository file exception
	 */
	@DeleteMapping(value = FILES_URL + "/delete/{fileUuid}")
	public Dataset removeFileOfDataset(@PathVariable("uuid") final UUID datasetUuid, @PathVariable("fileUuid") final UUID fileUuid) throws NotFoundElement,
			InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException, NoSuchRepositoryFileException {
		return datasetApiService.removeDatasetFile(datasetApiService.getDataset(datasetUuid, null), fileUuid);
	}

	/**
	 * Gets the list.
	 *
	 * @param datasetUuid the dataset uuid
	 * @return the list
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = FILES_URL + "/list")
	public List<RepositoryFile> getList(@PathVariable("uuid") final UUID datasetUuid) throws NotFoundElement {
		return datasetApiService.listDatasetFiles(datasetApiService.loadDataset(datasetUuid));
	}

	/**
	 * Update dataset file.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param metadata the metadata
	 * @return the dataset
	 * @throws NotFoundElement the not found element
	 * @throws InvalidRepositoryFileDataException the invalid repository file data
	 * exception
	 * @throws InvalidRepositoryPathException the invalid repository path exception
	 * @throws IOException Signals that an I/O exception has occurred.
	 * @throws NoSuchRepositoryFileException the no such repository file exception
	 */
	@PostMapping(value = FILES_URL + "/update")
	public Dataset updateDatasetFile(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final RepositoryFile metadata) throws NotFoundElement,
			InvalidRepositoryFileDataException, InvalidRepositoryPathException, IOException, NoSuchRepositoryFileException {
		return datasetApiService.updateDatasetFile(datasetApiService.getDataset(datasetUuid, null), metadata);
	}

	/**
	 * Creates the location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = LOCATION_URL + "/create")
	public DatasetLocation createLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocation datasetLocation) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		return datasetApiService.createLocation(dataset, datasetLocation);
	}

	/**
	 * Delete location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@RequestMapping(value = LOCATION_URL + "/delete", method = { RequestMethod.POST, RequestMethod.DELETE })
	public DatasetLocation deleteLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocation datasetLocation) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		return datasetApiService.removeLocation(dataset, datasetLocation);
	}

	/**
	 * List location.
	 *
	 * @param page the page
	 * @param uuid the uuid
	 * @return the page
	 */
	@GetMapping(value = LOCATION_URL + "/list")
	public Page<DatasetLocation> listLocation(@PathVariable("uuid") final UUID uuid, @ParameterObject final Pagination page) {
		return datasetApiService.listLocation(uuid, page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC, "id"));
	}

	/**
	 * Update location.
	 *
	 * @param datasetUuid the dataset uuid
	 * @param datasetLocation the dataset location
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@PostMapping(value = LOCATION_URL + "/update")
	public DatasetLocation updateLocation(@PathVariable("uuid") final UUID datasetUuid, @RequestBody final DatasetLocation datasetLocation) throws NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		return datasetApiService.updateLocation(dataset, datasetLocation);
	}

	/**
	 * Load by uuid.
	 *
	 * @param locationUuid the location uuid
	 * @return the dataset location
	 * @throws NotFoundElement the not found element
	 */
	@GetMapping(value = LOCATION_URL + "/{locationUuid}")
	public DatasetLocation loadLocationByUuid(@PathVariable("locationUuid") final UUID locationUuid) throws NotFoundElement {
		return datasetApiService.loadLocation(locationUuid);
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid}/download", method = RequestMethod.POST, params = { "mcpd" })
	public void downloadMcpd(@PathVariable("uuid") final UUID datasetUuid, HttpServletResponse response) throws IOException, NotFoundElement {
		final Dataset dataset = datasetApiService.getDataset(datasetUuid, null);
		if (dataset == null) {
			throw new NotFoundElement();
		}

		// Write MCPD to the stream.
		response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
		response.addHeader("Content-Disposition", String.format("attachment; filename=\"MCPD - %1s.xlsx\"", dataset.getUuid()));
		// response.flushBuffer();

		final OutputStream outputStream = response.getOutputStream();
		try {
			datasetApiService.writeXlsxMCPD(dataset, outputStream);
			response.flushBuffer();
		} catch (EOFException e) {
			LOG.warn("Download was aborted: {}", e.getMessage());
			throw e;
		}
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid}/download", method = RequestMethod.POST, params = { "metadata" })
	public void downloadMetadata(@PathVariable("uuid") final UUID datasetUuid, final HttpServletResponse response)
			throws IOException, NoSuchRepositoryFileException {

		var allFiles = datasetApiService.listDatasetFiles(datasetApiService.loadDataset(datasetUuid));
		var metadata = allFiles.stream().filter(rf -> rf.getOriginalFilename().equals(DATASET_METADATA_FILE_NAME)).findFirst().orElseThrow(() -> { throw new NotFoundElement(); });
		writeFileToOutputStream(metadata, response);
	}

	@DownloadEndpoint
	@RequestMapping(value = "/{uuid}/download/{fileId}", method = RequestMethod.POST)
	public void downloadDatasetFile(@PathVariable("uuid") final UUID datasetUuid, @PathVariable("fileId") final long fileId, final HttpServletResponse response)
			throws IOException, NoSuchRepositoryFileException {

		var allFiles = datasetApiService.listDatasetFiles(datasetApiService.loadDataset(datasetUuid));
		var datasetFile = allFiles.stream().filter(rf -> Objects.equals(rf.getId(), fileId)).findFirst().orElseThrow(() -> { throw new NotFoundElement(); });
		writeFileToOutputStream(datasetFile, response);
	}

	private void writeFileToOutputStream(final RepositoryFile file, final HttpServletResponse response) throws IOException, NoSuchRepositoryFileException {
		final byte[] data = repositoryService.getFileBytes(file);
		if (data != null) {
			response.setHeader(HttpHeaders.CACHE_CONTROL, "max-age=86400, s-maxage=86400, public, no-transform");
			response.setHeader(HttpHeaders.PRAGMA, "");
			response.setDateHeader(HttpHeaders.LAST_MODIFIED, file.getLastModifiedDate().toEpochMilli());
			response.setHeader(HttpHeaders.ETAG, file.getSha1Sum());
			response.setContentType(file.getContentType());
			response.addHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", file.getOriginalFilename()));

			response.setContentLength(data.length);
			response.getOutputStream().write(data);
		} else {
			throw new NoSuchRepositoryFileException("Bytes not available");
		}
		response.flushBuffer();
	}

	/**
	 * Create a new version of Dataset based on an existing published Dataset.
	 *
	 * @param uuid Dataset UUID
	 * @return the new version of Dataset
	 */
	@PostMapping(value = "/create-new-version")
	public Dataset createNewVersion(@RequestParam(value = "uuid", required = true) final UUID uuid) {
		final Dataset dataset = datasetApiService.loadDataset(uuid);
		return datasetApiService.createNewVersion(dataset);
	}

	@PostMapping(value = "/accessions-datasets")
	public Set<UUID> findDatasetsByAccessions(@RequestBody(required = false) AccessionFilter filter) {
		return datasetApiService.findAccessionsAmphibianDatasets(filter);
	}

	@PostMapping(path = { "/data" })
	@ApiOperation(value = "Read accessions observations from Datatables", notes = "")
	public Page<?> filterAccessionsDatasetsData(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Descriptor UUIDs", required = true) @RequestParam List<UUID> fields, // Fields UUIDs
		final Pagination page,
		@ApiParam(value = "Data filters and fields") @RequestBody(required = true) TraitFilters filter
	) throws Exception {

		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		var pagination = page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE);
		var request = new AccessionsDatasetsDataRequest();
		request.filters = filter;
		return amphibianService.getAccessionDatasetObservations(datasets, fields, request, pagination);
	}

	@PostMapping(path = { "/data/charts" })
	@ApiOperation(value = "getAccessionDatasetCharts", notes = "Read accession observation charts from Datatables")
	public List<ObservationChart> getAccessionDatasetCharts(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Descriptor UUIDs", required = true) @RequestParam List<UUID> fields, // Fields UUIDs
		@ApiParam(value = "Data filters and fields") @RequestBody(required = true) TraitFilters filter
	) throws SearchException {

		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		return amphibianService.getDatasetsCharts(datasets, fields, filter);
	}

	@PostMapping(path = { "/data/heatmap" })
	@ApiOperation(value = "getAccessionsObservationsHeatMap", notes = "Calculate a heat map of observations for two selected categorical descriptors in Datasets")
	public HeatMap getAccessionsObservationsHeatMap(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "X category field uuid") @RequestParam(name = "xCategory") UUID xCategoryField, // x category field uuid
		@ApiParam(value = "Y category field uuid") @RequestParam(name = "yCategory") UUID yCategoryField, // y category field uuid
		@ApiParam(value = "Data filters", required = true) @RequestBody(required = false) TraitFilters filter // data  filters
	) throws SearchException {
		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		var xCategoryDescriptor = descriptorApiService.getDescriptor(xCategoryField);
		var yCategoryDescriptor = descriptorApiService.getDescriptor(yCategoryField);
		if (xCategoryDescriptor == null || yCategoryDescriptor == null) {
			throw new NotFoundElement("No such categorical descriptor");
		}
		if (!xCategoryDescriptor.getDataType().equals(org.genesys.server.model.traits.Descriptor.DataType.CODED) 
			|| !yCategoryDescriptor.getDataType().equals(org.genesys.server.model.traits.Descriptor.DataType.CODED)) {
			
			throw new InvalidApiUsageException("Categorical Descriptor must have CODED data type");
		}
		return amphibianService.getDatasetsHeatMap(datasets, xCategoryDescriptor.getUuid(), yCategoryDescriptor.getUuid(), filter);
	}

	@PostMapping(path = { "/data/histogram" })
	@ApiOperation(value = "getObservationsHistograms", notes = "Get histogram of numerical observations from datasets")
	public List<ObservationHistogram> getObservationHistograms(
		@ApiParam(value = "Dataset UUIDs", required = true) @RequestParam List<UUID> datasetUuids, // Dataset UUIDs
		@ApiParam(value = "Specify fields to return") @RequestParam(name = "fields") List<UUID> fields, // selected fields
		@ApiParam(value = "Number of bins") @RequestParam(name = "bins", required = false, defaultValue = "4") Integer binsNumber, // number of bins
		@ApiParam(value = "Data filters", required = true) @RequestBody(required = false) TraitFilters filter // data  filters
	) throws SearchException {
		DatasetFilter datasetFilter = new DatasetFilter();
		datasetFilter.uuid(new HashSet<>(datasetUuids));
		var datasets = datasetService.list(datasetFilter, Pageable.ofSize(500)).getContent();
		return amphibianService.getDatasetsHistograms(datasets, fields, filter, binsNumber);
	}
}