DiversityTreeServiceImpl.java

/*
 * Copyright 2020 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.genesys.server.service.impl;

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.validator.routines.EmailValidator;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.filerepository.InvalidRepositoryFileDataException;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.NoSuchRepositoryFileException;
import org.genesys.filerepository.model.RepositoryFile;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.server.component.security.AsAdminInvoker;
import org.genesys.server.component.security.SecurityUtils;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.exception.SearchException;
import org.genesys.server.model.PublishState;
import org.genesys.server.model.UserRole;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.Crop;
import org.genesys.server.model.impl.DiversityTree;
import org.genesys.server.model.impl.DiversityTreeAccessionRef;
import org.genesys.server.model.impl.DiversityTreeCreator;
import org.genesys.server.model.impl.DiversityTreeVersions;
import org.genesys.server.model.impl.QDiversityTree;
import org.genesys.server.model.impl.QDiversityTreeAccessionRef;
import org.genesys.server.model.impl.QDiversityTreeCreator;
import org.genesys.server.persistence.DiversityTreeAccessionRefRepository;
import org.genesys.server.persistence.DiversityTreeCreatorRepository;
import org.genesys.server.persistence.DiversityTreeRepository;
import org.genesys.server.persistence.DiversityTreeVersionsRepository;
import org.genesys.server.service.CropService;
import org.genesys.server.service.DiversityTreeService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.filter.DiversityTreeFilter;
import org.genesys.server.service.worker.AccessionRefMatcher;
import org.genesys.util.HibernateUtil;
import org.genesys.util.JPAUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.dao.ConcurrencyFailureException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PostAuthorize;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.acls.domain.BasePermission;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.multipart.MultipartFile;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.types.dsl.BooleanExpression;

/**
 * @author Maxym Borodenko
 */
@Service
@Transactional(readOnly = true)
@Validated
public class DiversityTreeServiceImpl implements DiversityTreeService {

	private static final Logger LOG = LoggerFactory.getLogger(DiversityTreeServiceImpl.class);

	@Autowired
	private DiversityTreeRepository treeRepository;

	@Autowired
	private DiversityTreeAccessionRefRepository accessionRefRepository;

	@Autowired
	private DiversityTreeVersionsRepository treeVersionsRepository;

	@Autowired
	private DiversityTreeCreatorRepository creatorRepository;

	@Autowired
	private TaskExecutor taskExecutor;

	@Autowired
	private AccessionRefMatcher accessionRefMatcher;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired
	private CustomAclService aclService;

	@Autowired
	private CropService cropService;

	@Autowired
	private RepositoryService repositoryService;

	/** Execute code as admin */
	@Autowired
	private AsAdminInvoker asAdminInvoker;

	/** The securityUtils. */
	@Autowired
	private SecurityUtils securityUtils;

	private EmailValidator emailValidator = EmailValidator.getInstance();

	@Override
	public Page<DiversityTree> list(final DiversityTreeFilter filter, Pageable page) throws SearchException {
		BooleanBuilder published = new BooleanBuilder();
		published.and(QDiversityTree.diversityTree.state.eq(PublishState.PUBLISHED).and(QDiversityTree.diversityTree.current.isTrue()));

		if (filter.isFulltextQuery()) {
			return elasticsearchService.findAll(DiversityTree.class, filter, published, page);
		}
		published.and(filter.buildPredicate());

		return treeRepository.findAll(published, page);
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || #input.published || hasPermission(#input, 'READ')")
	public Page<DiversityTreeAccessionRef> listAccessionRefs(DiversityTree input, Pageable page) {
		input = get(input);
		return accessionRefRepository.findByList(input, page);
	}

	@Override
	public Set<DiversityTreeAccessionRef> loadAccessionRefs(Set<UUID> treeUuids, String nodeKey) {
		if (CollectionUtils.isEmpty(treeUuids) || StringUtils.isBlank(nodeKey)) {
			return Collections.emptySet();
		}

		String cleanedNodeKey = nodeKey.replaceAll("\\.+$", ""); //replace all dots at the end
		if (StringUtils.isNotBlank(cleanedNodeKey)) {
			QDiversityTreeAccessionRef accessionRef = QDiversityTreeAccessionRef.diversityTreeAccessionRef;
			BooleanExpression predicate = accessionRef.list().uuid.in(treeUuids).and(accessionRef.nodeKey.eq(cleanedNodeKey).or(accessionRef.nodeKey.startsWithIgnoreCase(cleanedNodeKey + ".")));
			return Sets.newHashSet(accessionRefRepository.findAll(predicate));
		}

		return Collections.emptySet();
	}

	@Override
	public long count(DiversityTreeFilter filter) throws SearchException {
		if (filter.isFulltextQuery()) {
			return elasticsearchService.count(DiversityTree.class, filter);
		}
		return treeRepository.count(filter.buildPredicate());
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || isAuthenticated()")
	public Page<DiversityTree> listForCurrentUser(DiversityTreeFilter filter, Pageable page) {
		Pageable markdownSortPageRequest = JPAUtils.toMarkdownSort(page, "title");

		if (securityUtils.hasRole(UserRole.ADMINISTRATOR) || securityUtils.hasRole(UserRole.DIVTREE)) {
			Page<DiversityTree> res = treeRepository.findAll(filter.buildPredicate(), markdownSortPageRequest);
			return new PageImpl<>(res.getContent(), page, res.getTotalElements());
		} else {
			final HashSet<Long> ids = new HashSet<>(securityUtils.listObjectIdentityIdsForCurrentUser(DiversityTree.class, BasePermission.WRITE));
			Page<DiversityTree> res = treeRepository.findAll(QDiversityTree.diversityTree.id.in(ids).and(filter.buildPredicate()), markdownSortPageRequest);
			return new PageImpl<>(res.getContent(), page, res.getTotalElements());
		}
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE')")
	public DiversityTree create(DiversityTree source) {
		LOG.info("Create DiversityTree.");

		final DiversityTreeVersions treeVersions = new DiversityTreeVersions();
		treeVersions.setCurrentVersion(null);
		treeVersionsRepository.save(treeVersions);

		final DiversityTree tree = new DiversityTree();
		copyValues(tree, source);
		tree.setCrop(validCropName(source.getCrop()));
		tree.setState(PublishState.DRAFT);
		tree.setVersions(treeVersions);
		tree.setCurrent(null);

		final DiversityTree loaded = treeRepository.save(tree);

		// Make DiversityTree publicly not-readable
		aclService.makePubliclyReadable(loaded, false);

		return lazyLoad(loaded);
	}

	@Override
	@PostAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || returnObject==null || returnObject.isPublished() || hasPermission(returnObject, 'READ')")
	public DiversityTree load(UUID uuid) {
		LOG.debug("Load DiversityTree.");
		return lazyLoad(get(uuid));
	}

	@Override
	@PostAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || returnObject==null || returnObject.isPublished() || hasPermission(returnObject, 'READ')")
	public DiversityTree get(UUID uuid) {
		DiversityTree tree = treeRepository.getByUuid(uuid);
		if (tree == null) {
			throw new NotFoundElement("Record not found by UUID=" + uuid);
		}
		return tree;
	}

	@Override
	@PostAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || returnObject==null || returnObject.isPublished() || hasPermission(returnObject, 'READ')")
	public DiversityTree get(UUID uuid, int version) {
		final DiversityTree tree = treeRepository.getByUuidAndVersion(uuid, version);
		if (tree == null) {
			throw new ConcurrencyFailureException("Record with that version doesn't exist");
		}
		return tree;
	}

	@Override
	public Map<String, ElasticsearchService.TermResult> getSuggestions(DiversityTreeFilter filter) throws SearchException, IOException {
		assert(filter != null);

		Set<String> suggestions = Sets.newHashSet("crop");
		Map<String, ElasticsearchService.TermResult> suggestionRes = new HashMap<>(suggestions.size());

		for (String suggestionKey: suggestions) {
			DiversityTreeFilter suggestionFilter = filter.copy(DiversityTreeFilter.class);
			suggestionFilter.state(PublishState.PUBLISHED);
			try {
				suggestionFilter.clearFilter(suggestionKey);
			} catch (NoSuchFieldException | IllegalAccessException e) {
				LOG.error("Error while clearing filter: ", e.getMessage());
			}
			suggestionFilter.current(true);

			ElasticsearchService.TermResult suggestion = elasticsearchService.termStatisticsAuto(DiversityTree.class, suggestionFilter, 100, suggestionKey);
			suggestionRes.put(suggestionKey, suggestion);
		}
		return suggestionRes;
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#source, 'WRITE')")
	public DiversityTree update(final DiversityTree source) {
		final DiversityTree loadedTree = getUnpublished(source);

		LOG.info("Updating DiversityTree.");
		loadedTree.setCrop(validCropName(source.getCrop()));
		copyValues(loadedTree, source);

		return lazyLoad(treeRepository.save(loadedTree));
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'DELETE')")
	public DiversityTree remove(DiversityTree input) {
		final DiversityTree loaded = getUnpublished(input);
		deleteAccessionRefs(loaded);
		treeRepository.delete(loaded);
		return loaded;
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'WRITE')")
	public DiversityTree addAccessionRefs(final DiversityTree input, final Collection<DiversityTreeAccessionRef> accessionRefs) {
		final DiversityTree loadedTree = getUnpublished(input.getUuid());

		LOG.info("Adding {} accession references to DiversityTree.", accessionRefs.size());
		Lists.partition(new ArrayList<>(accessionRefs), 10000).parallelStream().forEach(batch -> {
			batch.forEach(ref -> ref.setList(loadedTree)); // So that #equals works
			List<DiversityTreeAccessionRef> matchedRefs = accessionRefRepository.findExisting(loadedTree, batch);
			matchedRefs = accessionRefRepository.saveAll(matchedRefs);
			LOG.info("Stored {} accession references to DiversityTree.", batch.size());
		});


		LOG.info("Done saving {} accession refs to DiversityTree {}", accessionRefs.size(), loadedTree.getUuid());
		return lazyLoad(loadedTree);
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE')")
	public DiversityTree approve(final DiversityTree input) {
		final DiversityTree loaded = getUnpublished(input);

		loaded.setState(PublishState.PUBLISHED);

		// Make dataset publicly readable
		aclService.makePubliclyReadable(loaded, true);

		if (loaded.getTreeFile() != null) {
			// Relax permissions on divTree file: allow USERS and ANONYMOUS to read the file
			aclService.makePubliclyReadable(HibernateUtil.unproxy(loaded.getTreeFile()), true);
		}

		final DiversityTreeVersions versions = loaded.getVersions();
		final DiversityTree oldCurrentTree = versions.getAllVersions().stream().filter(s -> Objects.equals(s.getCurrent(), Boolean.TRUE)).findFirst().orElse(null);
		if (oldCurrentTree != null) {
			oldCurrentTree.setCurrent(null);
			treeRepository.save(oldCurrentTree);
		}
		loaded.setCurrent(Boolean.TRUE);
		loaded.setCurrentVersion(null);
		versions.setCurrentVersion(loaded);
		treeVersionsRepository.save(versions);

		return lazyLoad(treeRepository.save(loaded));
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'ADMINISTRATION')")
	public DiversityTree reject(final DiversityTree input) {
		DiversityTree loaded = get(input);

		if (loaded.isPublished() && !(securityUtils.hasRole(UserRole.ADMINISTRATOR) || securityUtils.hasRole(UserRole.DIVTREE))) {
			long oneDay = 24 * 60 * 60 * 1000;
			if (loaded.getLastModifiedDate() != null && loaded.getLastModifiedDate().toEpochMilli() <= (System.currentTimeMillis() - oneDay)) {
				throw new InvalidApiUsageException("Cannot be un-published. More than 24 hours have passed since the publication.");
			}
		}

		if (loaded.isPublished() && Objects.equals(loaded.getCurrent(), true)) {
			final DiversityTreeVersions treeVersions = loaded.getVersions();
			List<DiversityTree> notCurrentPublishedVersions = treeVersions.getAllVersions().stream()
					.filter(s -> s.getCurrent() == null && s.isPublished()).collect(Collectors.toList());

			if (!notCurrentPublishedVersions.isEmpty()) {
				UUID youngestTreeUUID = notCurrentPublishedVersions.stream()
						.max(Comparator.comparing(DiversityTree::getCreatedDate)).get().getUuid();

				loaded.setCurrent(null);
				treeRepository.save(loaded);

				DiversityTree youngestTree = treeRepository.getByUuid(youngestTreeUUID);
				youngestTree.setCurrent(true);
				treeRepository.save(youngestTree);
				treeVersions.setCurrentVersion(youngestTree);
				loaded.setCurrentVersion(youngestTreeUUID);
			} else {
				loaded.setCurrent(null);
				treeVersions.setCurrentVersion(null);
			}
			treeVersionsRepository.save(treeVersions);
		} else if (loaded.isPublished() && Objects.isNull(loaded.getCurrent())) {
			throw new InvalidApiUsageException("Cannot be un-published. The DiversityTree is not the latest version.");
		}

		loaded.setState(PublishState.DRAFT);

		// Make DiversityTree publicly not-readable
		aclService.makePubliclyReadable(loaded, false);

		if (loaded.getTreeFile() != null) {
			// Tighten permissions on divTree file
			aclService.makePubliclyReadable(HibernateUtil.unproxy(loaded.getTreeFile()), false);
		}

		return lazyLoad(treeRepository.save(loaded));
	}

	@Override
	public List<DiversityTree> listByAccession(Accession accession) {
		final BooleanExpression expression = QDiversityTree.diversityTree.state.in(PublishState.PUBLISHED)
				.and(QDiversityTree.diversityTree.accessionRefs.any().accession().eq(accession));

		return Lists.newArrayList(treeRepository.findAll(expression));
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'WRITE')")
	public DiversityTreeCreator createCreator(final DiversityTree input, final DiversityTreeCreator source) throws NotFoundElement {
		final DiversityTree loadedTree = getUnpublished(input);

		if (StringUtils.isNotBlank(source.getEmail()) && !emailValidator.isValid(source.getEmail())) {
			LOG.warn("Invalid email provided: {}", source.getEmail());
			throw new InvalidApiUsageException("Invalid email provided: " + source.getEmail());
		}

		source.setDiversityTree(loadedTree);
		return creatorRepository.save(source);
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'WRITE')")
	public DiversityTreeCreator updateCreator(DiversityTree input, final DiversityTreeCreator source) throws NotFoundElement {
		input = getUnpublished(input);

		final DiversityTreeCreator creator = loadCreator(source);
		if (!creator.getDiversityTree().getUuid().equals(input.getUuid())) {
			throw new InvalidApiUsageException("Creator does not belong to DiversityTreeCreator");
		}
		if (StringUtils.isNotBlank(source.getEmail()) && !emailValidator.isValid(source.getEmail())) {
			LOG.warn("Invalid email provided: {}", source.getEmail());
			throw new InvalidApiUsageException("Invalid email provided: " + source.getEmail());
		}
		copyValue(creator, source);
		return creatorRepository.save(creator);
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#tree, 'DELETE')")
	public DiversityTreeCreator removeCreator(DiversityTree tree, final DiversityTreeCreator input) throws NotFoundElement {
		tree = getUnpublished(tree);
		final DiversityTreeCreator creator = loadCreator(input);
		if (!creator.getDiversityTree().getUuid().equals(tree.getUuid())) {
			throw new InvalidApiUsageException("Creator does not belong to DiversityTree");
		}
		tree.getCreators().remove(creator);
		return creator;
	}

	@Override
	public Page<DiversityTreeCreator> listCreators(UUID treeUuid, Pageable page) {
		BooleanExpression expression = QDiversityTreeCreator.diversityTreeCreator.diversityTree().uuid.eq(treeUuid);
		return creatorRepository.findAll(expression, page);
	}

	@Override
	public DiversityTreeCreator loadCreator(DiversityTreeCreator input) throws NotFoundElement {
		return loadCreator(input.getUuid(), input.getVersion());
	}

	@Override
	public DiversityTreeCreator loadCreator(final UUID uuid) throws NotFoundElement {
		LOG.info("Load DiversityTreeCreator {}", uuid);
		final DiversityTreeCreator creator = creatorRepository.findByUuid(uuid);
		if (creator == null) {
			LOG.warn("DiversityTreeCreator {} not found", uuid);
			throw new NotFoundElement("DiversityTreeCreator by " + uuid.toString() + " not found");
		}
		return creator;
	}

	@Override
	public DiversityTreeCreator loadCreator(final UUID uuid, int version) throws NotFoundElement {
		final DiversityTreeCreator creator = creatorRepository.findByUuid(uuid);

		if (creator == null) {
			LOG.warn("DiversityTreeCreator by {} not found", uuid);
			throw new NotFoundElement("DiversityTreeCreator by " + uuid.toString() + " not found");
		}
		if (!creator.getVersion().equals(version)) {
			LOG.warn("Don't match the version");
			throw new ConcurrencyFailureException("Object version changed to " + creator.getVersion() + ", you provided " + version);
		}
		return creator;
	}

	@Override
	public List<DiversityTreeCreator> loadCreators(DiversityTree input) throws NotFoundElement {
		input = get(input);
		List<DiversityTreeCreator> creators = input.getCreators();
		creators.size();
		return creators;
	}

	@Override
	public List<DiversityTreeCreator> autocompleteCreators(String term) {
		BooleanExpression expression = QDiversityTreeCreator.diversityTreeCreator.fullName.startsWithIgnoreCase(term)
				.or(QDiversityTreeCreator.diversityTreeCreator.institutionalAffiliation.startsWithIgnoreCase(term));

		if (! (securityUtils.hasRole(UserRole.ADMINISTRATOR) || securityUtils.hasRole(UserRole.DIVTREE))) {
			final HashSet<Long> ids = new HashSet<>(securityUtils.listObjectIdentityIdsForCurrentUser(DiversityTree.class, BasePermission.WRITE));
			expression = expression.and(QDiversityTreeCreator.diversityTreeCreator.id.in(ids));
		}

		return creatorRepository.findAll(expression, PageRequest.of(0, 20, Sort.by("fullName"))).getContent();
	}

	@Override
	@Transactional(readOnly = true)
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE')")
	public void rematchAccessions() {
		treeRepository.findAll().forEach(this::rematchAccessions);
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#tree, 'WRITE')")
	public DiversityTree rematchAccessions(DiversityTree tree) {
		tree = treeRepository.getByUuid(tree.getUuid());
		if (tree == null) {
			return tree;
		}
		List<DiversityTreeAccessionRef> accessionRefs = tree.getAccessionRefs();
		tree.setAccessionCount(accessionRefs.size());
		tree = treeRepository.save(tree);

		LOG.info("Linking {} accessions with DiversityTree {}", tree.getAccessionCount(), tree.getId());

		batchRematchAccessionRefs(accessionRefs);

		LOG.info("Done scheduling of relinking {} accession refs.", accessionRefs.size());
		return lazyLoad(tree);
	}

	/**
	 * Schedule re-matching of AccessionRefs in batches
	 * @param accessionRefs
	 */
	private void batchRematchAccessionRefs(List<DiversityTreeAccessionRef> accessionRefs) {
		Lists.partition(accessionRefs, 10000).parallelStream().forEach((batch) -> {
			taskExecutor.execute(() -> {
				try {
					LOG.info("Rematching {} diversity trees refs", batch.size());
					// Thread.sleep(100);
					accessionRefMatcher.rematchAccessionRefs(batch, accessionRefRepository);
				} catch (Throwable e) {
					LOG.info("Rematch failed with {}", e.getMessage(), e);
				}
			});
		});
	}

	@Override
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE')")
	@Transactional(readOnly = false, propagation = Propagation.REQUIRED, isolation = Isolation.READ_UNCOMMITTED)
	public int clearAccessionRefs(Collection<Accession> accessions) {
		if (accessions == null || accessions.isEmpty()) {
			return 0;
		}

		Iterable<DiversityTreeAccessionRef> referencedRefs = accessionRefRepository.findAll(QDiversityTreeAccessionRef.diversityTreeAccessionRef.accession().in(accessions));
		AtomicInteger counter = new AtomicInteger();
		referencedRefs.forEach((ref) -> {
			ref.setAccession(null);
			counter.incrementAndGet();
		});
		accessionRefRepository.saveAll(referencedRefs);

		return counter.get();
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'WRITE')")
	public DiversityTree setAccessionRefs(final DiversityTree input, final Collection<DiversityTreeAccessionRef> accessionRefs) {
		final DiversityTree loaded = getUnpublished(input);

		LOG.info("Set accessions to DiversityTree {}. Input accessions {}", input.getUuid(), accessionRefs.size());
		deleteAccessionRefs(loaded);
		return addAccessionRefs(loaded, accessionRefs);
	}

	@Override
	@Transactional
	@PreAuthorize("(hasRole('ADMINISTRATOR') || hasPermission(#source, 'write')) && #source.published")
	public DiversityTree createNewVersion(DiversityTree source) {
		source = get(source);

		final DiversityTree tree = new DiversityTree();
		copyValues(tree, source);
		tree.setTreeFile(null);
		tree.setCrop(validCropName(source.getCrop()));
		tree.setState(PublishState.DRAFT);
		tree.setCurrent(null);
		tree.setUuid(null);
		tree.setVersions(source.getVersions());
		DiversityTree saved = treeRepository.save(tree);

		// Copy creators
		copyCreators(saved, source.getCreators());

		saved.setCurrentVersion(source.getUuid());

		// Make DiversityTree publicly not-readable
		aclService.makePubliclyReadable(saved, false);

		return saved;
	}

	@Override
	@Transactional
	@PreAuthorize("hasRole('ADMINISTRATOR') || hasRole('DIVTREE') || hasPermission(#input, 'WRITE')")
	public DiversityTree uploadFile(DiversityTree input, final MultipartFile file, final RepositoryFile metadata) throws IOException, InvalidRepositoryPathException, InvalidRepositoryFileDataException {
		DiversityTree divTree = getUnpublished(input);

		// Permit only a JSON file
		if (!file.getContentType().contains(MediaType.APPLICATION_JSON_VALUE)) {
			throw new InvalidApiUsageException("Invalid file type: " + file.getContentType() + " is not permitted.");
		}

		RepositoryFile treeFile = divTree.getTreeFile();

		// Update the original file if exists
		if (treeFile != null) {
			try {
				metadata.setUuid(treeFile.getUuid());
				metadata.setId(treeFile.getId());
				metadata.setVersion(treeFile.getVersion());
				treeFile = repositoryService.updateBytes(metadata, file.getContentType(), file.getInputStream());
				divTree.setTreeFile(treeFile);
			} catch (NoSuchRepositoryFileException e) {
				LOG.warn("Tree file not found: {}", e.getMessage(), e);
				treeFile = null;
			}
		}

		if (treeFile == null) {
			final Path folderPath = Paths.get("/divtree").toAbsolutePath();
			try {
				asAdminInvoker.invoke(() -> {
					// Ensure target folder exists for the DiversityTree
					return repositoryService.ensureFolder(folderPath);
				});
			} catch (Exception e) {
				LOG.warn("Could not create a folder: {}", e.getMessage());
			}

			treeFile = repositoryService.addFile(folderPath, divTree.getUuid().toString() + ".json", file.getContentType(), file.getInputStream(), metadata);
			divTree.setTreeFile(treeFile);
		}

		return lazyLoad(treeRepository.save(divTree));
	}

	private DiversityTree get(final DiversityTree input) {
		if (input == null || input.getId() == null) {
			throw new InvalidApiUsageException("Must be provided existing DiversityTree.");
		}

		final DiversityTree tree = treeRepository.findById(input.getId()).orElse(null);

		if (tree == null) {
			throw new NotFoundElement("Record not found by id=" + input.getId());
		}

		if (!tree.getVersion().equals(input.getVersion())) {
			LOG.warn("DiversityTree versions don't match anymore");
			throw new ConcurrencyFailureException("Object version changed to " + tree.getVersion() + ", you provided " + input.getVersion());
		}

		return tree;
	}

	private DiversityTree getUnpublished(final DiversityTree input) {
		DiversityTree loaded = get(input);
		if (loaded.isPublished()) {
			throw new InvalidApiUsageException("Cannot modify a published DiversityTree.");
		}
		return loaded;
	}

	private DiversityTree getUnpublished(final UUID uuid) {
		DiversityTree loaded = get(uuid);
		if (loaded.isPublished()) {
			throw new InvalidApiUsageException("Cannot modify a published DiversityTree.");
		}
		return loaded;
	}

	private String validCropName(final String cropName) {
		Crop crop = cropService.getCrop(cropName);
		if (crop == null) {
			throw new NotFoundElement("No such crop: " + cropName);
		}
		return crop.getShortName();
	}

	/**
	 * Deep load tree data.
	 *
	 * @param tree tree to deepload
	 * @return fully loaded tree
	 */
	private DiversityTree lazyLoad(final DiversityTree tree) {
		if (tree == null)
			throw new NotFoundElement("No such tree");

		if (tree.getCreators() != null)
			tree.getCreators().size();

		if (tree.getVersions() != null && tree.getVersions().getAllVersions() != null) {
			tree.getVersions().getAllVersions().size();
		}

		if (tree.getTreeFile() != null) {
			tree.getTreeFile().getId();
		}

		return tree;
	}

	/**
	 * Copy values.
	 *
	 * @param target the target
	 * @param source the source
	 */
	private void copyValues(final DiversityTree target, final DiversityTree source) {
		target.setTitle(source.getTitle());
		target.setState(source.getState());
		target.setDescription(source.getDescription());
		target.setPublisher(source.getPublisher());
		target.setDateCreated(source.getDateCreated());
		target.setUuid(source.getUuid());
		target.setDate(source.getDate());
		target.setSource(source.getSource());
		target.setTreeFile(source.getTreeFile());
	}

	/**
	 * Copy value.
	 *
	 * @param target the target
	 * @param source the source
	 */
	protected void copyValue(final DiversityTreeCreator target, final DiversityTreeCreator source) {
		target.setFullName(source.getFullName());
		target.setEmail(source.getEmail());
		target.setPhoneNumber(source.getPhoneNumber());
		target.setFax(source.getFax());
		target.setInstituteAddress(source.getInstituteAddress());
		target.setInstitutionalAffiliation(source.getInstitutionalAffiliation());
		target.setRole(source.getRole());
	}


	/**
	 * Copy and save DiversityTree creators.
	 *
	 * @param target the target
	 * @param creators the DiversityTree creators
	 */
	private void copyCreators(final DiversityTree target, final List<DiversityTreeCreator> creators) {
		if (creators == null || creators.size() == 0) {
			return;
		}

		List<DiversityTreeCreator> copiedCreators = Lists.newArrayList();
		creators.forEach(creator -> {
			DiversityTreeCreator copy = new DiversityTreeCreator();
			copyValue(copy, creator);
			copy.setDiversityTree(target);
			copiedCreators.add(copy);
		});
		target.setCreators(creatorRepository.saveAll(copiedCreators));
	}

	private void deleteAccessionRefs(final DiversityTree tree) {
		LOG.info("Removing {} accessionRefs from DiversityTree {}", tree.getAccessionRefs().size(), tree.getUuid());
		accessionRefRepository.deleteAll(tree.getAccessionRefs());
		tree.getAccessionRefs().clear();
	}
}