DatasetApiServiceImpl.java

/*
 * Copyright 2024 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.api.v2.facade.impl;

import java.net.MalformedURLException;
import java.net.URL;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.opencsv.CSVParser;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;

import lombok.extern.slf4j.Slf4j;

import org.apache.commons.lang3.time.StopWatch;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.server.api.v2.facade.DatasetApiService;
import org.genesys.server.api.v2.model.dataset.DatasetAccessionRefDTO;
import org.genesys.server.api.v2.model.dataset.DatasetCreatorDTO;
import org.genesys.server.api.v2.model.dataset.DatasetDTO;
import org.genesys.server.api.v2.model.dataset.DatasetInfo;
import org.genesys.server.api.v2.model.dataset.DatasetLangDTO;
import org.genesys.server.api.v2.model.dataset.DatasetLocationDTO;
import org.genesys.server.api.v2.model.dataset.TranslatedDatasetDTO;
import org.genesys.server.api.v2.model.dataset.TranslatedDatasetInfo;
import org.genesys.server.api.v2.model.impl.DescriptorDTO;
import org.genesys.server.api.v2.model.impl.RepositoryFileDTO;
import org.genesys.server.api.v2.model.impl.TranslatedDescriptorDTO;
import org.genesys.server.exception.DetailedConstraintViolationException;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.dataset.Dataset;
import org.genesys.server.model.dataset.DatasetAccessionRef;
import org.genesys.server.model.dataset.DatasetLang;
import org.genesys.server.model.filters.DatasetFilter;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.model.traits.Descriptor;
import org.genesys.server.model.traits.Descriptor.DataType;
import org.genesys.server.service.DatasetService;
import org.genesys.server.service.DatasetTranslationService;
import org.genesys.server.service.DescriptorService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.PartnerService;
import org.genesys.server.service.AmphibianService;
import org.genesys.server.service.AmphibianService.TraitFilters;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.taxonomy.gringlobal.component.CabReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.task.AsyncListenableTaskExecutor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletResponse;
import javax.validation.ConstraintViolation;
import javax.validation.Validator;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import static org.genesys.server.service.DatasetService.DATASET_METADATA_FILE_NAME;

@Service("datasetV2APIService")
@Validated
@Transactional(readOnly = true)
@Slf4j
public class DatasetApiServiceImpl extends APIFilteredTranslatedServiceFacadeImpl<DatasetService, DatasetDTO,
	TranslatedDatasetDTO, DatasetLangDTO, Dataset, DatasetLang,
	DatasetTranslationService.TranslatedDataset, DatasetFilter> implements DatasetApiService, InitializingBean {

	@Autowired
	private PartnerService partnerService;

	@Autowired
	private DescriptorService descriptorService;

	@Autowired
	private Validator validator;

	@Autowired
	private RepositoryService repositoryService;

	@Autowired
	private AmphibianService amphibianService;

	@Autowired
	private ObjectMapper objectMapper;

	@Value("${subsetting.traits.api:}")
	private URL subsettingTraitsApi;

	@Autowired(required = false)
	private AsyncListenableTaskExecutor taskExecutor;

	private SimpleClientHttpRequestFactory requestFactory;

	@Override
	public void afterPropertiesSet() throws Exception {
		this.requestFactory = new SimpleClientHttpRequestFactory();
		requestFactory.setBufferRequestBody(false);
		requestFactory.setConnectTimeout(2000);
		if (taskExecutor != null) requestFactory.setTaskExecutor(taskExecutor);
	}

	@Override
	protected TranslatedDatasetDTO convertTranslation(DatasetTranslationService.TranslatedDataset source) {
		return mapper.map(source);
	}

	@Override
	protected DatasetLang convertLang(DatasetLangDTO source) {
		return mapper.map(source);
	}

	@Override
	protected DatasetLangDTO convertLang(DatasetLang source) {
		return mapper.map(source);
	}

	@Override
	protected Dataset convert(DatasetDTO source) {
		return mapper.map(source);
	}

	@Override
	protected DatasetDTO convert(Dataset source) {
		return mapper.map(source);
	}

	@Override
	@Transactional
	public DatasetDTO forceUpdate(DatasetDTO dataset) {
		return mapper.map(service.forceUpdate(convert(dataset)));
	}

	@Override
	public List<TranslatedDescriptorDTO> loadDescriptors(UUID uuid) {
		return mapper.map(service.loadTranslatedDescriptors(service.getDataset(uuid)), mapper::map);
	}

	@Override
	@Transactional
	public DatasetDTO create(DatasetDTO dataset) {
		if (dataset.getOwner() == null || dataset.getOwner().getUuid() == null) {
			throw new InvalidApiUsageException("owner.uuid not provided");
		}
		dataset.setOwner(mapper.mapInfo(partnerService.get(dataset.getOwner().getUuid())));
		return super.create(dataset);
	}

	@Override
	@Transactional
	public DatasetDTO update(DatasetDTO dataset) {
		dataset.setOwner(mapper.mapInfo(partnerService.get(dataset.getOwner().getUuid())));
		return super.update(dataset);
	}

	@Override
	@Transactional
	public DatasetDTO setAccessionRefs(UUID uuid, int version, Collection<DatasetAccessionRef> accessionRefs) {
		Dataset dataset = service.getDataset(uuid, version);
		return mapper.map(service.setAccessionRefs(dataset, accessionRefs));
	}

	@Override
	@Transactional
	public DatasetDTO uploadAccessions(UUID uuid, int version, char separator, char quotechar, MultipartFile file) throws IOException {
		// Permit only a CSV file
		if (!file.getContentType().equalsIgnoreCase("text/csv")) {
			throw new InvalidApiUsageException("Invalid file type: " + file.getContentType() + " is not permitted.");
		}

		List<DatasetAccessionRef> accessionRefs = new ArrayList<>();

		// Build CSV parser
		CSVParser csvParser = new CSVParserBuilder().withSeparator(separator).withQuoteChar(quotechar).withEscapeChar((char) 0)
			.withStrictQuotes(false).withIgnoreLeadingWhiteSpace(false).withIgnoreQuotations(true).build();
		// Read file bytes as CSV
		try (CSVReader reader = new CSVReaderBuilder(CabReader.bomSafeReader(file.getInputStream())).withSkipLines(0).withCSVParser(csvParser).build()) {
			Iterator<DatasetAccessionRef> beanReader = CabReader.beanReader(DatasetAccessionRef.class, reader).iterator();
			DatasetAccessionRef acceRef = null;
			while (beanReader.hasNext() && (acceRef = beanReader.next()) != null) {
				Set<ConstraintViolation<DatasetAccessionRef>> violations = validator.validate(acceRef);
				if (violations == null || violations.isEmpty()) {
					accessionRefs.add(acceRef);
				} else {
					throw new DetailedConstraintViolationException("Failed to read CSV file in line " + reader.getLinesRead(), violations);
				}
			}
		}

		return setAccessionRefs(uuid, version, accessionRefs);
	}

	@Override
	@Transactional
	public DatasetDTO addDescriptors(UUID uuid, int version, final Set<UUID> descriptorUuids) {
		var dataset = service.getDataset(uuid, version);
		final Set<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorService.getDescriptor(descriptorUuid)).collect(Collectors.toSet());

		return mapper.map(service.addDescriptors(dataset, descriptors.toArray(new Descriptor[] {})));
	}

	@Override
	@Transactional
	public DatasetDTO removeDescriptors(UUID uuid, int version, Set<UUID> descriptorUuids) {
		final Dataset dataset = service.getDataset(uuid, version);
		final Set<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorService.getDescriptor(descriptorUuid)).collect(Collectors.toSet());
		return mapper.map(service.removeDescriptors(dataset, descriptors.toArray(new Descriptor[] {})));
	}

	@Override
	@Transactional
	public DatasetDTO updateDescriptors(UUID uuid, int version, List<UUID> descriptorUuids) {
		final Dataset dataset = service.getDataset(uuid, version);
		final List<Descriptor> descriptors = descriptorUuids.stream().map(descriptorUuid -> descriptorService.getDescriptor(descriptorUuid)).collect(Collectors.toList());
		return mapper.map(service.updateDescriptors(dataset, descriptors));
	}

	@Override
	public DatasetDTO loadDataset(DatasetDTO input) {
		return mapper.map(service.loadDataset(mapper.map(input)));
	}

	@Override
	public DatasetDTO loadDataset(UUID uuid, int version) {
		return mapper.map(service.loadDataset(uuid, version));
	}

	@Override
	public Page<DatasetAccessionRefDTO> listAccessions(UUID uuid, Pageable page) {
		var dataset = service.getDataset(uuid, null);
		return mapper.map(service.listAccessions(dataset, page), mapper::map);
	}

	@Override
	public List<DatasetInfo> listByAccession(Accession accession) {
		return mapper.map(service.listByAccession(accession), mapper::mapInfo);
	}

	@Override
	public Map<String, ElasticsearchService.TermResult> getSuggestions(DatasetFilter filter) throws SearchException, IOException {
		return service.getSuggestions(filter);
	}

	@Override
	public Page<DatasetInfo> listDatasetsForCurrentUser(DatasetFilter filter, Pageable page) throws SearchException {
		return mapper.map(service.listDatasetsForCurrentUser(filter, page), mapper::mapInfo);
	}

	@Override
	public DatasetDTO loadDataset(UUID uuid) throws NotFoundElement {
		return mapper.map(service.loadDataset(uuid));
	}

	@Override
	public TranslatedDatasetDTO loadTranslatedDataset(UUID uuid) throws NotFoundElement {
		return mapper.map(service.loadTranslatedDataset(uuid));
	}

	@Override
	@Transactional
	public DatasetDTO addDatasetFile(DatasetDTO dataset, MultipartFile file) throws NotFoundElement, IOException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
		return mapper.map(service.addDatasetFile(mapper.map(dataset), file));
	}

	@Override
	@Transactional
	public DatasetDTO removeDatasetFile(DatasetDTO dataset, UUID fileUuid) throws NotFoundElement, NoSuchRepositoryFileException, IOException {
		return mapper.map(service.removeDatasetFile(mapper.map(dataset), fileUuid));
	}

	@Override
	@Transactional
	public List<RepositoryFileDTO> listDatasetFiles(DatasetDTO dataset) throws NotFoundElement {
		return mapper.map(service.listDatasetFiles(mapper.map(dataset)), mapper::map);
	}

	@Override
	@Transactional
	public DatasetDTO addAccessionRefs(UUID uuid, int version, Set<DatasetAccessionRef> accessionRefs) throws NotFoundElement {
		final Dataset dataset = service.getDataset(uuid, version);
		log.info("Want to add {} accessionRefs to dataset {}", accessionRefs.size(), dataset.getUuid());
		return mapper.map(service.addAccessionRefs(dataset, accessionRefs));
	}

	@Override
	@Transactional
	public DatasetDTO reviewDataset(DatasetDTO dataset) {
		return mapper.map(service.reviewDataset(mapper.map(dataset)));
	}

	@Override
	@Transactional
	public DatasetDTO rejectDataset(DatasetDTO dataset) {
		return mapper.map(service.rejectDataset(mapper.map(dataset)));
	}

	@Override
	@Transactional
	public DatasetDTO approveDataset(DatasetDTO dataset) {
		return mapper.map(service.approveDataset(mapper.map(dataset)));
	}

	@Override
	@Transactional
	public DatasetDTO updateDatasetFile(DatasetDTO dataset, RepositoryFileDTO metadata) throws NoSuchRepositoryFileException {
		return mapper.map(service.updateDatasetFile(mapper.map(dataset), mapper.map(metadata)));
	}

	@Override
	public long countDatasets(DatasetFilter filter) throws SearchException {
		return service.countDatasets(filter);
	}

	@Override
	@Transactional
	public void rematchDatasetAccessions() {
		service.rematchDatasetAccessions();
	}

	@Override
	@Transactional
	public DatasetDTO rematchDatasetAccessions(UUID uuid, int version) {
		final Dataset dataset = service.getDataset(uuid, version);
		return mapper.map(service.rematchDatasetAccessions(dataset));
	}

	@Override
	@Transactional
	public DatasetCreatorDTO createDatasetCreator(UUID datasetUuid, DatasetCreatorDTO input) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.createDatasetCreator(dataset, mapper.map(input)));
	}

	@Override
	@Transactional
	public DatasetCreatorDTO removeDatasetCreator(UUID datasetUuid, DatasetCreatorDTO input) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.removeDatasetCreator(dataset, mapper.map(input)));
	}

	@Override
	public Page<DatasetCreatorDTO> listDatasetCreators(UUID datasetUuid, Pageable page) {
		return mapper.map(service.listDatasetCreators(datasetUuid, page), mapper::map);
	}

	@Override
	public Page<TranslatedDatasetInfo> listInfo(DatasetFilter filter, Pageable page) throws SearchException {
		return mapper.map(service.listTranslated(filter, page), mapper::mapInfo);
	}

	@Override
	public DatasetCreatorDTO loadDatasetCreator(UUID datasetCreatorUuid) throws NotFoundElement {
		return mapper.map(service.loadDatasetCreator(datasetCreatorUuid));
	}

	@Override
	@Transactional
	public DatasetCreatorDTO updateDatasetCreator(UUID datasetUuid, DatasetCreatorDTO datasetCreator) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.loadDatasetCreator(service.updateDatasetCreator(dataset, mapper.map(datasetCreator)).getUuid()));
	}

	@Override
	public List<DatasetCreatorDTO> autocompleteCreators(String text) {
		return mapper.map(service.autocompleteCreators(text), mapper::map);
	}

	@Override
	@Transactional
	public DatasetLocationDTO createLocation(UUID datasetUuid, DatasetLocationDTO input) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.createLocation(dataset, mapper.map(input)));
	}

	@Override
	@Transactional
	public DatasetLocationDTO removeLocation(UUID datasetUuid, DatasetLocationDTO input) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.removeLocation(dataset, mapper.map(input)));
	}

	@Override
	public Page<DatasetLocationDTO> listLocation(UUID datasetUuid, Pageable page) {
		return mapper.map(service.listLocation(datasetUuid, page), mapper::map);
	}

	@Override
	public DatasetLocationDTO loadLocation(UUID locationUuid) throws NotFoundElement {
		return mapper.map(service.loadLocation(locationUuid));
	}

	@Override
	@Transactional
	public DatasetLocationDTO updateLocation(UUID datasetUuid, DatasetLocationDTO input) throws NotFoundElement {
		final Dataset dataset = service.getDataset(datasetUuid, null);
		return mapper.map(service.updateLocation(dataset, mapper.map(input)));
	}

	@Override
	@Transactional
	public int clearAccessionRefs(Collection<Accession> accessions) {
		return service.clearAccessionRefs(accessions);
	}

	@Override
	public DatasetDTO getDataset(UUID uuid) {
		return mapper.map(service.getDataset(uuid));
	}

	@Override
	public DatasetDTO getDataset(UUID uuid, Integer version) {
		return mapper.map(service.getDataset(uuid, version));
	}

	@Override
	public void writeXlsxMCPD(DatasetDTO dataset, OutputStream outputStream) throws IOException {
		service.writeXlsxMCPD(mapper.map(dataset), outputStream);
	}

	@Override
	@Transactional
	public DatasetDTO createNewVersion(DatasetDTO source) {
		return mapper.map(service.createNewVersion(mapper.map(source)));
	}

	@Override
	@Transactional
	public long changeInstitute(FaoInstitute currentInstitute, FaoInstitute newInstitute) {
		return service.changeInstitute(currentInstitute, newInstitute);
	}

	@Override
	@Transactional
	public List<DescriptorDTO> synchronizeDescriptors(UUID uuid) {
		log.debug("Load Dataset by uuid {}", uuid);
		var dataset = service.loadDataset(uuid);
		return mapper.map(service.synchronizeDescriptors(dataset), mapper::map);
	}

	@Override
	@Transactional
	public void batchRematchAccessionRefs(List<DatasetAccessionRefDTO> accessionRefs) {
		service.batchRematchAccessionRefs(mapper.map(accessionRefs, mapper::map));
	}

	@Override
	public Set<UUID> findAccessionsAmphibianDatasets(AccessionFilter accessionFilter) {
		return service.findAccessionsAmphibianDatasets(accessionFilter);
	}

	@Override
	public void downloadMetadata(UUID datasetUuid, HttpServletResponse response) throws IOException, NoSuchRepositoryFileException {
		var allFiles = service.listDatasetFiles(service.loadDataset(datasetUuid));
		var metadata = allFiles.stream().filter(rf -> rf.getOriginalFilename().equals(DATASET_METADATA_FILE_NAME)).findFirst().orElseThrow(() -> {
			throw new NotFoundElement();
		});
		writeFileToOutputStream(metadata, response);
	}

	@Override
	public void downloadDatasetFile(UUID datasetUuid, long fileId, HttpServletResponse response) throws IOException, NoSuchRepositoryFileException {
		var allFiles = service.listDatasetFiles(service.loadDataset(datasetUuid));
		var datasetFile = allFiles.stream().filter(rf -> Objects.equals(rf.getId(), fileId)).findFirst().orElseThrow(() -> {
			throw new NotFoundElement();
		});
		writeFileToOutputStream(datasetFile, response);
	}

	private void writeFileToOutputStream(final RepositoryFile file, final HttpServletResponse response) throws IOException, NoSuchRepositoryFileException {
		final byte[] data = repositoryService.getFileBytes(file);
		if (data != null) {
			response.setHeader(HttpHeaders.CACHE_CONTROL, "max-age=86400, s-maxage=86400, public, no-transform");
			response.setHeader(HttpHeaders.PRAGMA, "");
			response.setDateHeader(HttpHeaders.LAST_MODIFIED, file.getLastModifiedDate().toEpochMilli());
			response.setHeader(HttpHeaders.ETAG, file.getSha1Sum());
			response.setContentType(file.getContentType());
			response.addHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", file.getOriginalFilename()));

			response.setContentLength(data.length);
			response.getOutputStream().write(data);
		} else {
			throw new NoSuchRepositoryFileException("Bytes not available");
		}
		response.flushBuffer();
	}

	@Override
	public Page<TranslatedDatasetDTO> list(DatasetFilter filter, Pageable page) throws SearchException {
		throw new InvalidApiUsageException("Not enabled!");
	}

	@Override
	public void coreCollection(Supplier<OutputStream> outputStreamSupplier, UUID datasetUuid, List<UUID> fields, TraitFilters filters, Map<?, ?> options) throws MalformedURLException {
		if (subsettingTraitsApi == null) {
			throw new RuntimeException("API not available");
		}
		callSubsettingApi(new URL(subsettingTraitsApi, "/api/v1/core-traits"), outputStreamSupplier, datasetUuid, fields, filters, options);
	}

	@Override
	public void clusterDataset(Supplier<OutputStream> outputStreamSupplier, UUID datasetUuid, List<UUID> fields, TraitFilters filters, Map<?, ?> options) throws MalformedURLException {
		if (subsettingTraitsApi == null) {
			throw new RuntimeException("API not available");
		}
		callSubsettingApi(new URL(subsettingTraitsApi, "/api/v1/traits-cluster"), outputStreamSupplier, datasetUuid, fields, filters, options);
	}

	/**
	 * Prepare request body and send everything to the specified endpoint. If response is 200 OK, stream the body to the {@code outputStreamSupplier}.
	 *
	 * @param endpoint Target endpoint
	 * @param outputStreamSupplier Get output stream only if response is 200 OK.
	 * @param datasetUuid Dataset UUID
	 * @param fields Selected dataset fields
	 * @param filters Dataset filters
	 * @param options Whatever options are provided by the UI
	 */
	private void callSubsettingApi(URL endpoint, Supplier<OutputStream> outputStreamSupplier, UUID datasetUuid, List<UUID> fields, TraitFilters filters, Map<?, ?> options) {

		var stopWatch = StopWatch.createStarted();
		var dataset = service.getDataset(datasetUuid);
		var descriptors = service.getDatasetDescriptors(dataset);
		log.debug("Dataset and {} descriptors loaded in {}ms", descriptors.size(), stopWatch.getTime(TimeUnit.MILLISECONDS));

		// Sanity checks
		var descriptorUuids = descriptors.stream().map(Descriptor::getUuid).collect(Collectors.toSet());
		// Check that requested descriptors match descriptors in the dataset
		assert(descriptorUuids.containsAll(fields));
		// Check that requested filters are for descriptors in the dataset
		assert(descriptorUuids.containsAll(filters.observations.keySet().stream().map(UUID::fromString).collect(Collectors.toSet())));

		var selectedDescriptors = descriptors.stream().filter(d -> fields.contains(d.getUuid())).collect(Collectors.toList());
		var NOMINAL_TYPES = Set.of(DataType.CODED, DataType.BOOLEAN);
		var QUANTITATIVE_TYPES = Set.of(DataType.NUMERIC, DataType.SCALE);
		var qualitativeDescriptors = selectedDescriptors.stream().filter(d -> NOMINAL_TYPES.contains(d.getDataType())).collect(Collectors.toList());
		var quantitativeDescriptors = selectedDescriptors.stream().filter(d -> QUANTITATIVE_TYPES.contains(d.getDataType())).collect(Collectors.toList());
		log.debug("Selected descriptors: {}", selectedDescriptors);
		selectedDescriptors.removeIf(d -> ! qualitativeDescriptors.contains(d) && ! quantitativeDescriptors.contains(d));
		log.debug("Clean selected descriptors: {}", selectedDescriptors);
		var traitsToLoad = selectedDescriptors.stream().map(Descriptor::getUuid).collect(Collectors.toList());
		var traitUuidsToLoad = traitsToLoad.stream().map(UUID::toString).collect(Collectors.toSet());

		log.debug("Requesting observation data for dataset {}: {}", datasetUuid, traitsToLoad);
		var observations = amphibianService.getObservations(dataset, Pageable.ofSize(200000), traitsToLoad, filters);
		log.debug("Observations for {} accessions and {} traits fetched at {}ms", observations.getContent().size(), traitsToLoad.size(), stopWatch.getTime(TimeUnit.MILLISECONDS));
		// try {
		// 	log.debug("Response from Amphibian:\n {}", objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(observations.getContent()));
		// } catch (JsonProcessingException e) {
		// }

		// Prepare data for clustering API
		var accessions = new LinkedList<String>();
		observations.getContent().forEach(accessionObservations -> {
			@SuppressWarnings("unchecked")
			var row = (Map<String, Object>) accessionObservations;
			accessions.add((String) row.get("accession"));
			row.keySet().removeIf(key -> ! Objects.equals("accession", key) && ! traitUuidsToLoad.contains(key));
		});

		var requestData = new LinkedHashMap<>();
		requestData.put("options", options);
		requestData.put("qualitativeDescriptors", qualitativeDescriptors.stream().map(Descriptor::getUuid).collect(Collectors.toList()));
		requestData.put("quantitativeDescriptors", quantitativeDescriptors.stream().map(Descriptor::getUuid).collect(Collectors.toList()));
		requestData.put("observations", observations.getContent());

		// Call clustering API
		try {
			if (log.isDebugEnabled()) {
				var requestBody = objectMapper.writeValueAsString(requestData);
				log.debug("Clustering API {} body:\n {}", endpoint, requestBody);
			}
			log.warn("Preparing API call to {} at {}ms", endpoint, stopWatch.getTime(TimeUnit.MILLISECONDS));

			var apiRequest = requestFactory.createRequest(endpoint.toURI(), HttpMethod.POST);
			var apiHeaders = apiRequest.getHeaders();
			apiHeaders.add("Content-Type", "application/json");
			apiHeaders.add("Accept", "application/json");
			objectMapper.writeValue(apiRequest.getBody(), requestData);

			log.warn("Calling {} at {}ms", endpoint, stopWatch.getTime(TimeUnit.MILLISECONDS));
			var response = apiRequest.execute();

			if (response.getStatusCode() == HttpStatus.OK) {
				log.warn("API {} responded at {}ms", endpoint, stopWatch.getTime(TimeUnit.MILLISECONDS));

				log.warn("Streaming response");
				response.getBody().transferTo(outputStreamSupplier.get());
				log.warn("Done streaming response at {}ms", stopWatch.getTime(TimeUnit.MILLISECONDS));
				return;

			} else {
				log.warn("API {} error {} at {}ms", endpoint, response.getStatusCode(), stopWatch.getTime(TimeUnit.MILLISECONDS));
				throw new RuntimeException("Clustering API responded with " + response.getStatusCode().name());
			}
		} catch (Throwable e) {
			log.error("Error calling {}: {}", endpoint, e.getMessage(), e);
			throw new InvalidApiUsageException("API error: " + e.getMessage(), e);
		}
	}

}