SubsetServiceImpl.java

/*
 * Copyright 2019 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.service.impl;

import static org.genesys.server.model.impl.QSubsetCreator.subsetCreator;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;

import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.security.model.AclSid;
import org.genesys.blocks.security.serialization.Permissions;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.server.component.aspect.NotifyForReview;
import org.genesys.server.component.aspect.NotifyOnPublished;
import org.genesys.server.component.security.SecurityUtils;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.Partner;
import org.genesys.server.model.PublishState;
import org.genesys.server.model.UserRole;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.model.impl.QSubset;
import org.genesys.server.model.impl.QSubsetAccessionRef;
import org.genesys.server.model.impl.Subset;
import org.genesys.server.model.impl.SubsetAccessionRef;
import org.genesys.server.model.impl.SubsetCreator;
import org.genesys.server.model.impl.SubsetLang;
import org.genesys.server.model.impl.SubsetVersions;
import org.genesys.server.persistence.FaoInstituteRepository;
import org.genesys.server.persistence.SubsetAccessionRefRepository;
import org.genesys.server.persistence.SubsetCreatorRepository;
import org.genesys.server.persistence.SubsetLangRepository;
import org.genesys.server.persistence.SubsetRepository;
import org.genesys.server.persistence.SubsetVersionsRepository;
import org.genesys.server.service.DownloadService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.SubsetService;
import org.genesys.server.service.SubsetTranslationService;
import org.genesys.server.service.SubsetTranslationService.TranslatedSubset;
import org.genesys.server.service.TranslatorService;
import org.genesys.server.service.TranslatorService.FormattedText;
import org.genesys.server.service.TranslatorService.TextFormat;
import org.genesys.server.service.TranslatorService.TranslatorException;
import org.genesys.server.service.filter.SubsetFilter;
import org.genesys.server.service.worker.AccessionRefMatcher;
import org.genesys.util.JPAUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.repository.support.Querydsl;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.acls.domain.BasePermission;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.core.types.dsl.PathBuilderFactory;
import com.querydsl.jpa.JPQLQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;

import static org.genesys.server.service.SubsetTranslationService.TranslatedSubset;

/**
 * The Class SubsetServiceImpl.
 *
 * @author Maxym Borodenko
 * @author Matija Obreza
 * @author Viacheslav Pavlov
 */
@Service
@Transactional(readOnly = true)
@Validated
public class SubsetServiceImpl
		extends FilteredTranslatedCRUDServiceImpl<Subset, SubsetLang, TranslatedSubset, SubsetFilter, SubsetRepository>
		implements SubsetService {

	private static final Logger LOG = LoggerFactory.getLogger(SubsetServiceImpl.class);

	@PersistenceContext
	private EntityManager entityManager;

	@Autowired
	private SubsetRepository subsetRepository;

	/** The subset versions repository. */
	@Autowired
	private SubsetVersionsRepository subsetVersionsRepository;

	@Autowired
	private FaoInstituteRepository instituteRepository;

	@Autowired
	private SubsetCreatorRepository subsetCreatorRepository;

	/** The securityUtils. */
	@Autowired
	private SecurityUtils securityUtils;

	@Autowired
	private SubsetAccessionRefRepository accessionRefRepository;

	@Autowired
	private TaskExecutor taskExecutor;

	@Autowired
	private CustomAclService aclService;

	/** The download service. */
	@Autowired
	private DownloadService downloadService;

	@Autowired
	private AccessionRefMatcher accessionRefMatcher;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired(required = false)
	private TranslatorService translatorService;

	@Autowired
	private JPAQueryFactory jpaQueryFactory;

	private Comparator<SubsetAccessionRef> distinctAcceRefsComparator;

	@Override
	public void afterPropertiesSet() throws Exception {
		distinctAcceRefsComparator = Comparator
				.comparing((SubsetAccessionRef ref) -> ref.getSubset().getId())
				.thenComparing(SubsetAccessionRef::getInstCode)
				.thenComparing(SubsetAccessionRef::getGenus)
				.thenComparing(SubsetAccessionRef::getAcceNumb);
	}

	@Component(value = "SubsetTranslationSupport")
	protected static class SubsetTranslationSupport
			extends BaseTranslationSupport<Subset, SubsetLang, TranslatedSubset, SubsetFilter, SubsetLangRepository>
			implements SubsetTranslationService {

		@Override
		@Transactional
		@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#source.entity.id, 'org.genesys.server.model.impl.Subset', 'WRITE')")
		public SubsetLang create(SubsetLang source) {
			return super.create(source);
		}

		@Override
		@Transactional
		@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#target.entity.id, 'org.genesys.server.model.impl.Subset', 'WRITE')")
		public SubsetLang update(SubsetLang updated, SubsetLang target) {
			return super.update(updated, target);
		}

		@Override
		@Transactional
		@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#entity.entity.id, 'org.genesys.server.model.impl.Subset', 'WRITE')")
		public SubsetLang remove(SubsetLang entity) {
			return super.remove(entity);
		}

	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#target, 'WRITE')")
	public Subset updateFast(Subset updated, Subset target) {
		if (target.isPublished()) {
			throw new InvalidApiUsageException("Cannot modify a published Subset.");
		}

		copyValues(target, updated);
		translationSupport.deleteTranslation(target, updated.getOriginalLanguageTag()); // Remove translation of original
																																										// language
		return subsetRepository.save(target);
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#source.owner, 'CREATE')")
	public Subset createFast(Subset source) {
		LOG.info("Create Subset.");

		final SubsetVersions subsetVersions = new SubsetVersions();
		subsetVersions.setCurrentVersion(null);
		subsetVersionsRepository.save(subsetVersions);

		final Subset subset = new Subset();
		copyValues(subset, source);
		subset.setState(PublishState.DRAFT);
		subset.setVersions(subsetVersions);
		subset.setCurrent(null);

		final Subset loaded = subsetRepository.save(subset);

		// Make Subset publicly not-readable
		aclService.makePubliclyReadable(loaded, false);

		// Grant all permissions to the Partner's SID
		final AclSid sid = aclService.ensureAuthoritySid(loaded.getOwner().getAuthorityName());
		aclService.setPermissions(loaded, sid, new Permissions().grantAll());

		return loaded;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Page<Subset> list(final SubsetFilter filter, final Pageable page) throws SearchException {
		final BooleanBuilder published = new BooleanBuilder();
		published.and(QSubset.subset.state.eq(PublishState.PUBLISHED).and(QSubset.subset.current.isTrue()));
		if (filter.isFulltextQuery()) {
			return elasticsearchService.findAll(Subset.class, filter, published, page);
		}
		published.and(filter.buildPredicate());
		return subsetRepository.findAll(published, page);
	}

	@Override
	public Page<TranslatedSubset> listFiltered(SubsetFilter filter, Pageable page) throws SearchException {
		SubsetFilter published = new SubsetFilter();
		published.state = Set.of(PublishState.PUBLISHED);
		published.current = true;

		// Assemble filter
		if (filter != null) {
			filter = filter.copy(SubsetFilter.class); // Don't modify incoming filter
			var f = filter;
			while (f.AND != null) f = f.AND; // Find empty AND so we don't override incoming values
			f.AND(published);
		} else {
			filter = published;
		}

		return super.listFiltered(filter, page);
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') || #subset.published || hasPermission(#subset, 'READ')")
	public Page<SubsetAccessionRef> listAccessions(Subset subset, final Pageable page) {
		subset = getSubset(subset);
		return accessionRefRepository.findByList(subset, page);
	}

	@Override
	public long countSubsets(SubsetFilter filter) throws SearchException {
		if (filter.isFulltextQuery()) {
			return elasticsearchService.count(Subset.class, filter);
		}
		return subsetRepository.count(filter.buildPredicate());
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') or isAuthenticated()")
	public Page<Subset> listSubsetsForCurrentUser(SubsetFilter filter, Pageable page) {
		Pageable markdownSortPageRequest = JPAUtils.toMarkdownSort(page, "title");

		if (securityUtils.hasRole(UserRole.ADMINISTRATOR)) {
			Page<Subset> res = subsetRepository.findAll(filter.buildPredicate(), markdownSortPageRequest);
			return new PageImpl<>(res.getContent(), page, res.getTotalElements());
		} else {
			final HashSet<Long> partnersIds = new HashSet<>(
					securityUtils.listObjectIdentityIdsForCurrentUser(Partner.class, BasePermission.WRITE));
			Page<Subset> res = subsetRepository
					.findAll(QSubset.subset.owner().id.in(partnersIds).and(filter.buildPredicate()), markdownSortPageRequest);
			return new PageImpl<>(res.getContent(), page, res.getTotalElements());
		}
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#source.owner, 'CREATE')")
	public Subset create(final Subset source) {
		LOG.info("Create Subset.");

		final SubsetVersions subsetVersions = new SubsetVersions();
		subsetVersions.setCurrentVersion(null);
		subsetVersionsRepository.save(subsetVersions);

		final Subset subset = new Subset();
		copyValues(subset, source);
		subset.setState(PublishState.DRAFT);
		subset.setVersions(subsetVersions);
		subset.setCurrent(null);

		final Subset loaded = subsetRepository.save(subset);

		// Make Subset publicly not-readable
		aclService.makePubliclyReadable(loaded, false);

		// Grant all permissions to the Partner's SID
		final AclSid sid = aclService.ensureAuthoritySid(loaded.getOwner().getAuthorityName());
		aclService.setPermissions(loaded, sid, new Permissions().grantAll());

		return lazyLoad(loaded);
	}

	/**
	 * Copy values.
	 *
	 * @param target the target
	 * @param source the source
	 */
	private void copyValues(final Subset target, final Subset source) {
		target.setOwner(source.getOwner());
		target.setWiewsCode(source.getWiewsCode());
		target.setInstitute(instituteRepository.findByCode(source.getWiewsCode()));
		target.setState(source.getState());
		target.setOriginalLanguageTag(StringUtils.defaultIfBlank(source.getOriginalLanguageTag(), "en")); // API v1 doesn't
																																																			// handle
																																																			// originalLanguage
		target.setTitle(source.getTitle());
		target.setDescription(source.getDescription());
		target.setPublisher(source.getPublisher());
		target.setDateCreated(source.getDateCreated());
		target.setUuid(source.getUuid());
		target.setDate(source.getDate());
		target.setSource(source.getSource());
		target.setSubsetType(source.getSubsetType());
		target.setSelectionMethod(source.getSelectionMethod());

		if (source.getCrops() != null) {
			if (target.getCrops() == null) {
				target.setCrops(new HashSet<>());
			}
			target.getCrops().clear();
			target.getCrops().addAll(source.getCrops());
		}
	}

	/**
	 * Deep load subset data.
	 *
	 * @param subset subset to deepload
	 * @return fully loaded subset
	 */
	private Subset lazyLoad(final Subset subset) {
		if (subset == null)
			throw new NotFoundElement("No such subset");

		if (subset.getInstitute() != null) {
			subset.getInstitute().getId();
		}
		if (subset.getOwner() != null) {
			subset.getOwner().getId();
		}
		if (subset.getCrops() != null) {
			subset.getCrops().size();
		}
		if (subset.getCreators() != null) {
			subset.getCreators().size();
		}

		if (subset.getVersions() != null && subset.getVersions().getAllVersions() != null) {
			subset.getVersions().getAllVersions().size();
		}

		return subset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Subset loadSubset(final Subset input) {
		LOG.debug("Load Subset.");
		return lazyLoad(getSubset(input));
	}

	@Override
	public TranslatedSubset loadTranslated(UUID uuid) {
		Subset subset = getSubset(uuid);
		return translationSupport.getTranslated(subset);
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.genesys.server.service.SubsetService#loadSubset(java.util.UUID)
	 */
	@Override
	@PostAuthorize("hasRole('ADMINISTRATOR') || returnObject==null || returnObject.isPublished() || hasPermission(returnObject, 'READ')")
	public Subset loadSubset(UUID uuid) {
		return lazyLoad(getSubset(uuid));
	}

	private Subset getSubset(final Subset input) {
		final Subset subset = subsetRepository.findById(input.getId()).orElse(null);

		if (subset == null) {
			throw new NotFoundElement("Record not found by id=" + input.getId());
		}

		if (!subset.getVersion().equals(input.getVersion())) {
			LOG.warn("Subset versions don't match anymore");
			throw new ConcurrencyFailureException(
					"Object version changed to " + subset.getVersion() + ", you provided " + input.getVersion());
		}

		return subset;
	}

	private Subset getUnpublishedSubset(final Subset input) {
		Subset loadedSubset = getSubset(input);
		if (loadedSubset.isPublished()) {
			throw new InvalidApiUsageException("Cannot modify a published Subset.");
		}
		return loadedSubset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Subset getSubset(final UUID uuid) {
		Subset subset = subsetRepository.getByUuid(uuid);
		if (subset == null) {
			throw new NotFoundElement("Record not found by UUID=" + uuid);
		}
		return subset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@PostAuthorize("hasRole('ADMINISTRATOR') || returnObject==null || returnObject.isPublished() || hasPermission(returnObject, 'READ')")
	public Subset getSubset(final UUID uuid, final int version) {
		final Subset subset = subsetRepository.getByUuidAndVersion(uuid, version);
		if (subset == null) {
			throw new ConcurrencyFailureException("Record with that version doesn't exist");
		}
		return subset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public Map<String, ElasticsearchService.TermResult> getSuggestions(SubsetFilter filter)
			throws SearchException, IOException {
		assert (filter != null);

		Set<String> suggestions = Sets.newHashSet("crops");
		Map<String, ElasticsearchService.TermResult> suggestionRes = new HashMap<>(suggestions.size());

		for (String suggestionKey : suggestions) {
			SubsetFilter suggestionFilter = filter.copy(SubsetFilter.class);
			suggestionFilter.state(PublishState.PUBLISHED);
			try {
				suggestionFilter.clearFilter(suggestionKey);
			} catch (NoSuchFieldException | IllegalAccessException e) {
				LOG.error("Error while clearing filter: ", e.getMessage());
			}
			suggestionFilter.current(true);

			ElasticsearchService.TermResult suggestion = elasticsearchService.termStatisticsAuto(Subset.class,
					suggestionFilter, 100, suggestionKey);
			suggestionRes.put(suggestionKey, suggestion);
		}
		return suggestionRes;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#source, 'WRITE')")
	public Subset update(final Subset source) {
		final Subset loadedSubset = getUnpublishedSubset(source);
		LOG.info("Updating Subset.");

		return update(source, loadedSubset);
	}

	@Override
	public Subset update(Subset updated, Subset target) {
		copyValues(target, updated);
		translationSupport.deleteTranslation(target, updated.getOriginalLanguageTag()); // Remove translation of original language
		return lazyLoad(subsetRepository.save(target));
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#input, 'DELETE')")
	public Subset delete(final Subset input) {
		final Subset loadedSubset = getUnpublishedSubset(input);
		deleteAccessionRefs(loadedSubset);
		subsetRepository.delete(loadedSubset);
		return loadedSubset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#input, 'WRITE')")
	public Subset setAccessionRefs(final Subset input, final Collection<SubsetAccessionRef> accessionRefs) {
		final Subset loadedSubset = getUnpublishedSubset(input);

		LOG.info("Set accessions to Subset {}. Input accessions {}", input.getUuid(), accessionRefs.size());
		deleteAccessionRefs(loadedSubset);
		return addAccessionRefs(loadedSubset, accessionRefs);
	}

	private void deleteAccessionRefs(final Subset loadedSubset) {
		Lists.partition(loadedSubset.getAccessionRefs(), 10000).parallelStream().forEach(batch -> {
			accessionRefRepository.deleteAll(batch);
			LOG.debug("Removed {} accessionRefs of Subset {} from database", batch.size(), loadedSubset.getUuid());
		});
		LOG.info("Removed {} accessionRefs from Subset {}", loadedSubset.getAccessionRefs().size(), loadedSubset.getUuid());
		loadedSubset.getAccessionRefs().clear();
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#input, 'WRITE')")
	public Subset addAccessionRefs(final Subset input, final Collection<SubsetAccessionRef> accessionRefs) {
		final Subset loadedSubset = getUnpublishedSubset(input.getUuid());

		final var existingRefCount = accessionRefRepository.countByList(loadedSubset);
		LOG.info("Adding {} accession references to Subset.", accessionRefs.size());

		accessionRefs.forEach(ref -> ref.setSubset(loadedSubset)); // So that #equals works
		Lists.partition(new ArrayList<>(getDistinctAccessionRefs(accessionRefs)), 10000).parallelStream().forEach(batch -> {
			List<SubsetAccessionRef> updatedRefs;
			if (existingRefCount == 0) {
				updatedRefs = accessionRefRepository.saveAllAndFlush(batch);
				LOG.warn("Added new {} accession references to Subset.", updatedRefs.size());
			} else {
				updatedRefs = accessionRefRepository.findExisting(loadedSubset, batch);
				updatedRefs = accessionRefRepository.saveAllAndFlush(updatedRefs);
				LOG.info("Stored {} accession references to Subset.", batch.size());
			}
			// Rematching is done in AccessionRefAspect!
		});

		loadedSubset.setAccessionCount(accessionRefRepository.countByList(loadedSubset));
		final var updatedSubset = subsetRepository.save(loadedSubset);
		LOG.info("Done saving {} accession refs to subset {} has count={}", accessionRefs.size(), updatedSubset.getUuid(),
				updatedSubset.getAccessionCount());
		return lazyLoad(loadedSubset);
	}

	private Subset getUnpublishedSubset(UUID uuid) {
		Subset loadedSubset = getSubset(uuid);
		if (loadedSubset.isPublished()) {
			throw new InvalidApiUsageException("Cannot modify a published Subset.");
		}
		return loadedSubset;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR')")
	@NotifyOnPublished
	public Subset approveSubset(final Subset subset) {
		final Subset loadedSubset = getUnpublishedSubset(subset);

		if (loadedSubset.getState() == PublishState.DRAFT) {
			throw new InvalidApiUsageException("Subset should be sent for review before publication");
		}
		loadedSubset.setState(PublishState.PUBLISHED);

		// Make dataset publicly readable
		aclService.makePubliclyReadable(loadedSubset, true);

		final SubsetVersions subsetVersions = loadedSubset.getVersions();
		final Subset oldCurrentSubset = subsetVersions.getAllVersions().stream()
				.filter(s -> Objects.equals(s.getCurrent(), Boolean.TRUE)).findFirst().orElse(null);
		if (oldCurrentSubset != null) {
			oldCurrentSubset.setCurrent(null);
			subsetRepository.save(oldCurrentSubset);
		}
		loadedSubset.setCurrent(Boolean.TRUE);
		loadedSubset.setCurrentVersion(null);
		subsetVersions.setCurrentVersion(loadedSubset);
		subsetVersionsRepository.save(subsetVersions);

		return lazyLoad(subsetRepository.save(loadedSubset));
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	@NotifyForReview
	public Subset reviewSubset(final Subset subset) {
		final Subset loadedSubset = getUnpublishedSubset(subset);

		if (loadedSubset.getState() == PublishState.REVIEWING) {
			throw new InvalidApiUsageException("The subset is already under approval");
		}
		loadedSubset.setState(PublishState.REVIEWING);

		return lazyLoad(subsetRepository.save(loadedSubset));
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'ADMINISTRATION')")
	public Subset rejectSubset(final Subset subset) {
		Subset loadedSubset = getSubset(subset);

		if (loadedSubset.isPublished() && !securityUtils.hasRole(UserRole.ADMINISTRATOR)) {
			long oneDay = 24 * 60 * 60 * 1000;
			if (loadedSubset.getLastModifiedDate() != null
					&& loadedSubset.getLastModifiedDate().toEpochMilli() <= (System.currentTimeMillis() - oneDay)) {
				throw new InvalidApiUsageException(
						"Cannot be un-published. More than 24 hours have passed since the publication.");
			}
		}

		if (loadedSubset.isPublished() && Objects.equals(loadedSubset.getCurrent(), true)) {
			final SubsetVersions subsetVersions = loadedSubset.getVersions();
			List<Subset> notCurrentPublishedVersions = subsetVersions.getAllVersions().stream()
					.filter(s -> s.getCurrent() == null && s.isPublished()).collect(Collectors.toList());

			if (!notCurrentPublishedVersions.isEmpty()) {
				UUID youngestSubsetUUID = notCurrentPublishedVersions.stream()
						.max(Comparator.comparing(Subset::getCreatedDate)).get().getUuid();

				loadedSubset.setCurrent(null);
				subsetRepository.save(loadedSubset);

				Subset youngestSubset = subsetRepository.getByUuid(youngestSubsetUUID);
				youngestSubset.setCurrent(true);
				subsetRepository.save(youngestSubset);
				subsetVersions.setCurrentVersion(youngestSubset);
				loadedSubset.setCurrentVersion(youngestSubsetUUID);
			} else {
				loadedSubset.setCurrent(null);
				subsetVersions.setCurrentVersion(null);
			}
			subsetVersionsRepository.save(subsetVersions);
		} else if (loadedSubset.isPublished() && Objects.isNull(loadedSubset.getCurrent())) {
			throw new InvalidApiUsageException("Cannot be un-published. The subset is not the latest version.");
		}

		loadedSubset.setState(PublishState.DRAFT);

		// Make Subset publicly not-readable
		aclService.makePubliclyReadable(loadedSubset, false);

		return lazyLoad(subsetRepository.save(loadedSubset));
	}

	@Override
	public List<Subset> listByAccession(Accession accession) {
		return (List<Subset>) subsetRepository.findAll(QSubset.subset.state.in(PublishState.PUBLISHED)
				.and(QSubset.subset.accessionRefs.any().accession().eq(accession)));
	}

	@Transactional
	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public SubsetCreator createSubsetCreator(Subset subset, SubsetCreator input) throws NotFoundElement {
		final Subset loadedSubset = getUnpublishedSubset(subset);

		input.setSubset(loadedSubset);
		return subsetCreatorRepository.save(input);
	}

	@Transactional
	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public SubsetCreator removeSubsetCreator(Subset subset, SubsetCreator input) throws NotFoundElement {
		subset = getUnpublishedSubset(subset);

		final SubsetCreator subsetCreator = loadSubsetCreator(input);
		if (!subsetCreator.getSubset().getUuid().equals(subset.getUuid())) {
			throw new InvalidApiUsageException("Creator does not belong to subset");
		}
		subset.getCreators().remove(subsetCreator);
		return subsetCreator;
	}

	@Transactional
	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public SubsetCreator removeSubsetCreator(Subset subset, UUID subsetCreatorUuid) throws NotFoundElement {
		subset = getUnpublishedSubset(subset);
		final SubsetCreator subsetCreator = loadSubsetCreator(subsetCreatorUuid);
		if (!subsetCreator.getSubset().getUuid().equals(subset.getUuid())) {
			throw new InvalidApiUsageException("Creator does not belong to subset");
		}
		subset.getCreators().remove(subsetCreator);
		return subsetCreator;
	}

	@Override
	@PreAuthorize("subset.isPublished or hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public List<SubsetCreator> listSubsetCreators(Subset subset) {
		return subsetCreatorRepository.listByUUidOfSubset(subset.getUuid());
	}

	@Override
	public SubsetCreator loadSubsetCreator(SubsetCreator input) throws NotFoundElement {
		final SubsetCreator subsetCreator = subsetCreatorRepository.findByUuid(input.getUuid());

		if (subsetCreator == null) {
			LOG.error("SubsetCreator {} not found", input);
			throw new org.genesys.server.exception.NotFoundElement(
					"SubsetCreator by " + input.getUuid().toString() + " no found");
		}
		if (!subsetCreator.getVersion().equals(input.getVersion())) {
			LOG.error("Don't match the version");
			throw new ConcurrencyFailureException(
					"Object version changed to " + subsetCreator.getVersion() + ", you provided " + input.getVersion());
		}
		return subsetCreator;
	}

	@Override
	public SubsetCreator loadSubsetCreator(UUID subsetCreatorUuid) throws NotFoundElement {
		LOG.info("Load SubsetCreator {}", subsetCreatorUuid);
		final SubsetCreator subsetCreator = subsetCreatorRepository.findByUuid(subsetCreatorUuid);
		if (subsetCreator == null) {
			LOG.error("SubsetCreator {} not found", subsetCreatorUuid);
			throw new org.genesys.server.exception.NotFoundElement(
					"SubsetCreator by " + subsetCreatorUuid.toString() + " no found");
		}
		return subsetCreator;
	}

	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public SubsetCreator updateSubsetCreator(Subset subset, SubsetCreator input) throws NotFoundElement {
		subset = getUnpublishedSubset(subset);

		final SubsetCreator subsetCreator = loadSubsetCreator(input);
		if (!subsetCreator.getSubset().getUuid().equals(subset.getUuid())) {
			throw new InvalidApiUsageException("Creator does not belong to subset");
		}
		copyValue(subsetCreator, input);
		return subsetCreatorRepository.save(subsetCreator);
	}

	/**
	 * Copy value.
	 *
	 * @param target the target
	 * @param source the source
	 */
	protected void copyValue(final SubsetCreator target, final SubsetCreator source) {
		target.setFullName(source.getFullName());
		target.setEmail(source.getEmail());
		target.setPhoneNumber(source.getPhoneNumber());
		target.setFax(source.getFax());
		target.setInstituteAddress(source.getInstituteAddress());
		target.setInstitutionalAffiliation(source.getInstitutionalAffiliation());
		target.setRole(source.getRole());
	}

	@Override
	public List<SubsetCreator> autocompleteCreators(String text) {
		final HashSet<Long> ids = new HashSet<>(
				securityUtils.listObjectIdentityIdsForCurrentUser(Subset.class, BasePermission.WRITE));
		final Predicate predicate = subsetCreator.subset().id.in(ids)
				.and(subsetCreator.fullName.startsWithIgnoreCase(text).or(subsetCreator.institutionalAffiliation
						.startsWithIgnoreCase(text)));
		return subsetCreatorRepository.findAll(predicate, PageRequest.of(0, 20, Sort.by("fullName"))).getContent();
	}

	@Override
	@Transactional(readOnly = true)
	@PreAuthorize("hasRole('ADMINISTRATOR')")
	public void rematchSubsetAccessions() {
		subsetRepository.findAll().forEach(subset -> {
			rematchSubsetAccessions(subset);
		});
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#subset, 'WRITE')")
	public Subset rematchSubsetAccessions(Subset subset) {
		subset = subsetRepository.getByUuid(subset.getUuid());
		if (subset == null) {
			return subset;
		}
		List<SubsetAccessionRef> accessionRefs = subset.getAccessionRefs();
		subset.setAccessionCount(accessionRefs.size());
		subset = subsetRepository.save(subset);

		LOG.info("Linking {} accessions with subset {}", subset.getAccessionCount(), subset.getId());

		batchRematchAccessionRefs(accessionRefs);

		LOG.info("Done scheduling of relinking {} accession refs.", accessionRefs.size());
		return lazyLoad(subset);
	}

	/**
	 * Schedule re-matching of AccessionRefs in batches
	 * 
	 * @param accessionRefs
	 */
	@Override
	public void batchRematchAccessionRefs(List<SubsetAccessionRef> accessionRefs) {
		Lists.partition(accessionRefs, 10000).parallelStream().forEach((batch) -> {
			taskExecutor.execute(() -> {
				try {
					LOG.info("Rematching {} subset refs", batch.size());
					// Thread.sleep(100);
					accessionRefMatcher.rematchAccessionRefs(batch, accessionRefRepository);
				} catch (Throwable e) {
					LOG.info("Rematch failed with {}", e.getMessage(), e);
				}
			});
		});
	}

	@Override
	@Transactional(readOnly = false, propagation = Propagation.REQUIRED, isolation = Isolation.READ_UNCOMMITTED)
	public int clearAccessionRefs(Collection<Accession> accessions) {
		if (accessions == null || accessions.isEmpty()) {
			return 0;
		}

		Iterable<SubsetAccessionRef> referencedRefs = accessionRefRepository
				.findAll(QSubsetAccessionRef.subsetAccessionRef.accession().in(accessions));
		AtomicInteger counter = new AtomicInteger();
		referencedRefs.forEach((ref) -> {
			ref.setAccession(null);
			counter.incrementAndGet();
		});
		accessionRefRepository.saveAll(getDistinctAccessionRefs(Lists.newArrayList(referencedRefs)));

		return counter.get();
	}

	@Override
	public void writeXlsxMCPD(Subset subset, OutputStream outputStream) throws IOException {
		PathBuilder<Accession> builder = new PathBuilderFactory().create(Accession.class);
		Querydsl querydsl = new Querydsl(entityManager, builder);
		JPQLQuery<Long> queryAccessionId = querydsl.createQuery(QSubsetAccessionRef.subsetAccessionRef)
				// select id only
				.select(QSubsetAccessionRef.subsetAccessionRef.accession().id)
				// order by id
				.orderBy(QSubsetAccessionRef.subsetAccessionRef.accession().id.asc());

		// Apply where
		queryAccessionId.where(QSubsetAccessionRef.subsetAccessionRef.list().eq(subset));

		downloadService.writeXlsxMCPD(queryAccessionId, outputStream, "", "/subsets/" + subset.getUuid());
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	@Transactional
	@PreAuthorize("(hasRole('ADMINISTRATOR') || hasPermission(#source, 'WRITE')) && #source.published")
	public Subset createNewVersion(Subset source) {
		source = getSubset(source);

		final Subset subset = new Subset();
		copyValues(subset, source);
		subset.setState(PublishState.DRAFT);
		subset.setCurrent(null);
		subset.setUuid(null);
		subset.setVersions(source.getVersions());
		Subset saved = subsetRepository.save(subset);

		// Copy accessionRefs
		copyAccessionRefs(saved, source.getAccessionRefs());

		// Copy creators
		copyCreators(saved, source.getCreators());

		saved.setCurrentVersion(source.getUuid());

		// Make Subset publicly not-readable
		aclService.makePubliclyReadable(saved, false);

		// Grant all permissions to the Partner's SID
		final AclSid sid = aclService.ensureAuthoritySid(saved.getOwner().getAuthorityName());
		aclService.setPermissions(saved, sid, new Permissions().grantAll());

		return saved;
	}

	/**
	 * Copy and save subset accessionRefs.
	 *
	 * @param target        the target
	 * @param accessionRefs the subset accessionRefs
	 * @return
	 */
	private Subset copyAccessionRefs(final Subset target, final List<SubsetAccessionRef> accessionRefs) {
		if (accessionRefs == null || accessionRefs.size() == 0) {
			return target;
		}

		final Subset loadedSubset = getSubset(target);
		List<SubsetAccessionRef> copiedAccessionRefs = Lists.newArrayList();
		getDistinctAccessionRefs(accessionRefs).forEach(sAccessionRef -> {
			SubsetAccessionRef copy = new SubsetAccessionRef();
			copyAccessionRef(copy, sAccessionRef);
			copy.setSubset(loadedSubset);
			copiedAccessionRefs.add(copy);
		});
		accessionRefRepository.saveAll(copiedAccessionRefs);

		loadedSubset.setAccessionCount((int) accessionRefRepository.countByList(loadedSubset));
		LOG.info("Done saving {} accession refs, have {} in subset", accessionRefs.size(),
				loadedSubset.getAccessionCount());
		return subsetRepository.save(loadedSubset);
	}

	/**
	 * Copy and save subset creators.
	 *
	 * @param target   the target
	 * @param creators the subset creators
	 */
	private void copyCreators(final Subset target, final List<SubsetCreator> creators) {
		if (creators == null || creators.size() == 0) {
			return;
		}

		List<SubsetCreator> copiedCreators = Lists.newArrayList();
		creators.forEach(creator -> {
			SubsetCreator copy = new SubsetCreator();
			copyValue(copy, creator);
			copy.setSubset(target);
			copiedCreators.add(copy);
		});
		target.setCreators(subsetCreatorRepository.saveAll(copiedCreators));
	}

	/**
	 * Copy subset accessionRef values.
	 *
	 * @param target the target
	 * @param source the source
	 */
	private void copyAccessionRef(final SubsetAccessionRef target, final SubsetAccessionRef source) {
		target.setDoi(source.getDoi());
		target.setInstCode(source.getInstCode());
		target.setAcceNumb(source.getAcceNumb());
		target.setGenus(source.getGenus());
		target.setSpecies(source.getSpecies());
		target.setAccession(source.getAccession());
	}

	@Override
	@Transactional(propagation = Propagation.MANDATORY) // Need to be part of an existing transaction!
	@PreAuthorize("hasRole('ADMINISTRATOR')")
	public long changeInstitute(FaoInstitute currentInstitute, FaoInstitute newInstitute) {
		LOG.warn("Migrating subsets from {} to {}", currentInstitute.getCode(), newInstitute.getCode());

		// Update accession references
		var qAccessionRef = QSubsetAccessionRef.subsetAccessionRef;
		var updatedRefs = jpaQueryFactory.update(qAccessionRef)
				// Update instCode
				.set(qAccessionRef.instCode, newInstitute.getCode())
				// WHERE
				.where(qAccessionRef.instCode.eq(currentInstitute.getCode()))
				// Execute
				.execute();

		LOG.warn("Updated {} subset accession refs from {} to {}", updatedRefs, currentInstitute.getCode(),
				newInstitute.getCode());

		// Update Subsets
		var qSubset = QSubset.subset;
		return jpaQueryFactory.update(qSubset)
				// Update instCode
				.set(qSubset.institute(), newInstitute)
				// WHERE
				.where(qSubset.institute().eq(currentInstitute))
				// Execute
				.execute();
	}

	private Set<SubsetAccessionRef> getDistinctAccessionRefs(Collection<SubsetAccessionRef> accessionRefs) {
		var distinctRefs = new TreeSet<>(distinctAcceRefsComparator);
		distinctRefs.addAll(accessionRefs);
		return distinctRefs;
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#original, 'write')")
	public SubsetLang machineTranslate(Subset original, String targetLanguage) throws TranslatorException {
		if (Objects.equals(original.getOriginalLanguageTag(), targetLanguage)) {
			throw new InvalidApiUsageException("Source and target language are the same");
		}

		var mt = new SubsetLang();
		mt.setMachineTranslated(true);
		mt.setLanguageTag(targetLanguage);
		mt.setEntity(original);

		if (translatorService == null) return mt;

		var builder = TranslatorService.TranslationStructuredRequest.builder()
			.targetLang(targetLanguage);

		if (! Objects.equals(Locale.ENGLISH.getLanguage(), targetLanguage) && ! Objects.equals(Locale.ENGLISH.getLanguage(), original.getOriginalLanguageTag())) {
			// Translations to other languages use the English version (either original or translated)
			var enTranslation = translationSupport.getLang(original, Locale.ENGLISH.getLanguage());
			if (enTranslation == null) {
				throw new InvalidApiUsageException("English text is not available.");
			}

			builder
				.sourceLang(enTranslation.getLanguageTag())
				// .context(enTranslation.getDescription())
				.texts(Map.of(
					"title", new FormattedText(TextFormat.markdown, enTranslation.getTitle()),
					"selectionMethod", new FormattedText(TextFormat.markdown, enTranslation.getSelectionMethod()),
					"description", new FormattedText(TextFormat.markdown, enTranslation.getDescription())
				));

		} else {
			// Translations to English use the original texts
			builder
				.sourceLang(original.getOriginalLanguageTag())
				// .context(original.getDescription())
				.texts(Map.of(
					"title", new FormattedText(TextFormat.markdown, original.getTitle()),
					"selectionMethod", new FormattedText(TextFormat.markdown, original.getSelectionMethod()),
					"description", new FormattedText(TextFormat.markdown, original.getDescription())
				));
		}

		var translations = translatorService.translate(builder.build());

		if (StringUtils.isNotBlank(original.getTitle())) {
			mt.setTitle(translations.getTexts().get("title"));
		}
		if (StringUtils.isNotBlank(original.getSelectionMethod())) {
			mt.setSelectionMethod(translations.getTexts().get("selectionMethod"));
		}
		if (StringUtils.isNotBlank(original.getDescription())) {
			mt.setDescription(translations.getTexts().get("description"));
		}
		return mt;
	}
}