AccessionUploader.java

/*
 * Copyright 2018 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.genesys.server.service.worker;

import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import javax.validation.ConstraintViolation;
import javax.validation.ConstraintViolationException;
import javax.validation.ValidationException;
import javax.validation.Validator;

import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.text.WordUtils;
import org.genesys.blocks.model.BasicModel;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.server.api.PleaseRetryException;
import org.genesys.server.api.model.Api1Constants;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.model.Partner;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.genesys.AccessionAlias;
import org.genesys.server.model.genesys.AccessionAlias.AliasType;
import org.genesys.server.model.genesys.AccessionCollect;
import org.genesys.server.model.genesys.AccessionHistoric;
import org.genesys.server.model.genesys.AccessionId;
import org.genesys.server.model.genesys.AccessionRemark;
import org.genesys.server.model.genesys.PDCI;
import org.genesys.server.model.genesys.SelfCopy;
import org.genesys.server.model.genesys.Taxonomy2;
import org.genesys.server.model.impl.Country;
import org.genesys.server.model.impl.Crop;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.persistence.AccessionHistoricRepository;
import org.genesys.server.persistence.AccessionIdRepository;
import org.genesys.server.persistence.AccessionRepository;
import org.genesys.server.persistence.FaoInstituteRepository;
import org.genesys.server.service.CropService;
import org.genesys.server.service.GeoService;
import org.genesys.server.service.TaxonomyService;
import org.genesys.server.service.worker.AccessionOpResponse.UpsertResult;
import org.genesys.util.PDCICalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Isolation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ReflectionUtils;

import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.google.common.collect.Iterators;


/**
 * The Class AccessionUploader.
 */
@Component
@Transactional
public class AccessionUploader implements InitializingBean {
	private static final Logger LOG = LoggerFactory.getLogger(AccessionUploader.class);

	private ObjectMapper objectMapper;
	
	@Autowired
	private AccessionRepository accessionRepository;

	@Autowired
	private AccessionIdRepository accessionIdRepository;

	@Autowired
	private AccessionHistoricRepository accessionHistoricRepository;

	@Autowired
	private TaxonomyService taxonomyService;

	@Autowired
	private FaoInstituteRepository faoInstituteRepository;

	@Autowired
	private AccessionCounter accessionCounter;

	@Autowired
	private GeoService geoService;

	@Autowired
	private CropService cropService;

	@Autowired
	private Validator validator;

	private static final String[] ACCESSIONID_FIELDS = { Api1Constants.Accession.BREDCODE, Api1Constants.Accession.BREDNAME, Api1Constants.Accession.STORAGE, Api1Constants.Accession.DUPLSITE, Api1Constants.Accession.LATITUDE, Api1Constants.Accession.LONGITUDE, Api1Constants.Accession.ELEVATION, Api1Constants.Accession.COORDUNCERT, Api1Constants.Accession.COORDDATUM, Api1Constants.Accession.GEOREFMETH };

	@Override
	public void afterPropertiesSet() throws Exception {
		objectMapper = new ObjectMapper();
		objectMapper.enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT);
		// ignore stuff we don't understand
		objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
		// explicit json views: every fields needs to be annotated, therefore enabled
		// objectMapper.enable(MapperFeature.DEFAULT_VIEW_INCLUSION);

		// JSR310 java.time
		var javaTimeModule = new JavaTimeModule();
		objectMapper.registerModule(javaTimeModule);

		// Custom deserializers
		SimpleModule testModule = new SimpleModule("Custom Deserializers");
		testModule.addDeserializer(AccessionRemark.class, new JsonDeserializers.AccessionRemarkDeserializer(AccessionRemark.class));
		objectMapper.registerModule(testModule);
	}

	/**
	 * Upsert accessions.
	 *
	 * @param institute the fao institute
	 * @param updates the updates
	 * @throws IOException
	 */
	@Transactional(timeout = 250, rollbackFor = { Exception.class, RuntimeException.class, Error.class })
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE')")
	public List<AccessionOpResponse> upsertAccessions(FaoInstitute institute, ArrayNode updates) throws InvalidApiUsageException, ConstraintViolationException, ValidationException {
		assert (updates.isArray());
		StopWatch stopWatch = StopWatch.createStarted();
		ObjectReader reader = objectMapper.readerFor(Accession.class);

		Instant batchDate = Instant.now();
		final List<Accession> accessions = new ArrayList<>(updates.size());
		final List<AccessionOpResponse> responses = new ArrayList<>(updates.size());
		List<Accession> toSave = new ArrayList<>(updates.size());

		// Data provider, if configured
		Partner owner = institute.getOwner();
		// The required string prefix for dataProviderId values (e.g. "EURISCO:")
		String dataProviderIdPrefix = owner == null ? null : owner.getShortName() + ":";

		for (JsonNode accn : updates) {
			LOG.trace("Received: {}", accn);
			try {
				var taxonomyField = accn.get("taxonomy");
				if (taxonomyField != null) {
					var genus = taxonomyField.get("genus");
					if (genus == null || StringUtils.isBlank(genus.asText())) {
						throw new InvalidApiUsageException("Blank taxonomy genus is not accepted");
					}
					var newGenus = taxonomyField.get("newGenus");
					if (newGenus != null && StringUtils.isBlank(newGenus.asText())) {
						throw new InvalidApiUsageException("Blank taxonomy genus is not accepted");
					}
				}
				unwrapGeoIfNeeded((ObjectNode) accn);

				Accession accession = reader.readValue(accn);
				if (accession.getInstituteCode() == null) {
					throw new InvalidApiUsageException("instituteCode is missing in " + accn);
				}
				if (! institute.getCode().equals(accession.getInstituteCode())) {
					throw new InvalidApiUsageException("Accession does not belong to institute " + institute.getCode());
				}
				if (accession.getDoi() == null && accession.getAccessionNumber() == null) {
					throw new InvalidApiUsageException("accessionNumber OR doi missing in " + accn);
				}
				if (accession.getDataProviderId() != null) {
					if (owner == null) {
						// if using dataProviderId, the institute must be linked to one data provider
						throw new InvalidApiUsageException("dataProviderId is not supported for institutes without owner");
					}
					if (! accession.getDataProviderId().startsWith(dataProviderIdPrefix)) {
						// ... and it must have the correct prefix
						throw new InvalidApiUsageException("dataProviderId must begin with '" + dataProviderIdPrefix + "'");
					}
				}

				accessions.add(accession);
				responses.add(new AccessionOpResponse(accession));

			} catch (IOException e) {
				LOG.error("Could not parse input: {}", e.getMessage(), e);
				accessions.add(null); // need to match get(index) with responses and updates
				responses.add(new AccessionOpResponse(accn)
					.setResult(new UpsertResult(UpsertResult.Type.ERROR)).setError(e.getMessage()));
			} catch (InvalidApiUsageException e) {
				LOG.warn("Input error: {}", e.getMessage());
				accessions.add(null); // need to match get(index) with responses and updates
				responses.add(new AccessionOpResponse(accn)
					.setResult(new UpsertResult(UpsertResult.Type.ERROR)).setError(e.getMessage()));
			}
		}

		LOG.debug("Processed incoming JSON for {} accessions in {}ms", updates.size(), stopWatch.getTime());

		List<Accession> existingAccessions = accessionRepository.find(! institute.hasUniqueAcceNumbs(), accessions);
		LOG.debug("Have {} accessions for update and {} exist in {}ms", accessions.size(), existingAccessions.size(), stopWatch.getTime());

		int errorCount = 0;

		for (int i = 0; i < accessions.size(); i++) {
			final JsonNode update = updates.get(i);

			// Should we?
			final Accession forU = accessions.get(i);
			if (forU == null) {
				LOG.debug("Skipping update of {}", update);
				continue;
			}

			final Accession updateA = accessions.get(i);
			AccessionOpResponse response = responses.get(i);
			LOG.trace("Upserting: {}", update);

			Accession accession = findExistingAccession(institute, existingAccessions, updateA);

			if (accession == null) {
				// LOG.trace("New accesson");
				LOG.trace("New accession {} not in {}", update, existingAccessions);
				accession = new Accession();
				accession.setAccessionId(new AccessionId());
				// Assign UUID on insert
				accession.getAccessionId().setUuid(UUID.randomUUID());;
				accession.setInstitute(institute);
				accession.setDataProviderId(updateA.getDataProviderId());
				response.setResult(new UpsertResult(UpsertResult.Type.INSERT).setUUID(accession.getUuid()));
			} else {
				LOG.trace("Updating accession {}", accession);
				response.setResult(new UpsertResult(UpsertResult.Type.UPDATE).setUUID(accession.getUuid()));
			}

			try {

				// Accessions with registered DOI require the DOI to be provided
				if (accession.getDoi() != null && updateA.getDoi() == null) {
					throw new InvalidApiUsageException("DOI not included for accession with registered DOI=" + accession.getDoi());
				}

				//
				// Allow changing the holding institute when dataProviderId is used
				if (! accession.getInstitute().getId().equals(institute.getId())) {
					if (accession.getDataProviderId() == null) {
						throw new InvalidApiUsageException("Not changing institute, existing accession does not specify dataProviderId");
					} else if (! accession.getDataProviderId().equals(updateA.getDataProviderId())) {
						throw new InvalidApiUsageException("Not changing institute, dataProviderId does not match " + accession.getDataProviderId());
					} else {
						if (SecurityContextUtil.hasRole("ADMINISTRATOR") || SecurityContextUtil.hasPermission(accession.getInstitute(), "WRITE")) {
							LOG.trace("Changing instutute for {} to {}", accession, institute);
							accessionCounter.recountInstitute(accession.getInstitute());
							accession.setInstitute(institute);
						} else {
							throw new InvalidApiUsageException("Cannot change instituteCode without WRITE permission");
						}
					}
				}

				accession = applyChanges(update, updateA, accession);
				LOG.trace("Applied changes to 1 accession after {}ms", stopWatch.getTime());

				accession.setLastModifiedDate(batchDate);

				PDCI pdci = accession.getAccessionId().getPdci();
				if (pdci == null) {
					pdci = new PDCI();
					pdci.setAccession(accession.getAccessionId());
					accession.getAccessionId().setPdci(pdci);
				}
				PDCICalculator.updatePdci(pdci, accession);
				LOG.trace("Updated PDCI to 1 accession after {}ms", stopWatch.getTime());

				toSave.add(validate(accession));
			} catch (PleaseRetryException e) {
				throw e;
			} catch (InvalidApiUsageException e) {
				if (LOG.isDebugEnabled()) {
					LOG.error(e.getMessage(), e);
				} else {
					LOG.error(e.getMessage());
				}
				if (errorCount++ == 0) {
					LOG.error("Failed to upsert JSON: {} {}", e.getMessage(), update);
				}
				response.setError(e.getMessage());
			}
		}

		LOG.debug("Ready to update {} accessions after {}ms", toSave.size(), stopWatch.getTime());
		toSave = accessionRepository.saveAll(toSave);
		LOG.trace("Saved: {}", toSave);
		accessionCounter.recountInstitute(institute);

		LOG.info("Upserted {} accessions in {}ms", updates.size(), stopWatch.getTime());
		return responses;
	}

	private void unwrapGeoIfNeeded(final ObjectNode accn) {
		final var geo = (ObjectNode) accn.get("geo");
		
		if (geo != null) {
			moveJsonValue(geo, accn, Api1Constants.Geo.LATITUDE, Api1Constants.Accession.LATITUDE);
			moveJsonValue(geo, accn, Api1Constants.Geo.LONGITUDE, Api1Constants.Accession.LONGITUDE);
			moveJsonValue(geo, accn, Api1Constants.Geo.ELEVATION, Api1Constants.Accession.ELEVATION);
			moveJsonValue(geo, accn, Api1Constants.Geo.COORDUNCERT, Api1Constants.Accession.COORDUNCERT);
			moveJsonValue(geo, accn, Api1Constants.Geo.COORDDATUM, Api1Constants.Accession.COORDDATUM);
			moveJsonValue(geo, accn, Api1Constants.Geo.GEOREFMETH, Api1Constants.Accession.GEOREFMETH);
			accn.remove("geo");
		}
	}

//	private void upgradeRename(ObjectNode update, String v1name, String v2name) {
//		if (update.has(v1name)) {
//			update.set(v2name, update.remove(v1name));
//		}
//	}

	private void moveJsonValue(ObjectNode source, ObjectNode target, final String originalName, final String newName) {
		if (source.has(originalName)) {
			if (target.has(newName)) {
				source.remove(originalName);
			} else {
				target.set(newName, source.remove(originalName));
			}
		}
	}

	private Accession findExistingAccession(FaoInstitute institute, List<Accession> existingAccessions, final Accession updateA) {
		return existingAccessions.stream().filter(existing -> {
			return
				// by DOI
				(existing.getDoi() != null && existing.getDoi().equals(updateA.getDoi()))
				// or by UUID
				|| (updateA.getUuid() != null && updateA.getUuid().equals(existing.getUuid()))
				// or by dataProviderId
				|| (updateA.getDataProviderId() != null && updateA.getDataProviderId().equals(existing.getDataProviderId()))
				// or by pair
				|| (updateA.getUuid() == null && institute.hasUniqueAcceNumbs() && (StringUtils.equalsIgnoreCase(existing.getInstituteCode(), updateA.getInstituteCode()) && StringUtils.equalsIgnoreCase(existing.getAccessionNumber(),
				updateA.getAccessionNumber())))
				// or by triplet
				|| (updateA.getUuid() == null && StringUtils.equalsIgnoreCase(existing.getInstituteCode(), updateA.getInstituteCode()) && StringUtils.equalsIgnoreCase(existing.getAccessionNumber(),
				updateA.getAccessionNumber()) && updateA.getTaxonomy() != null && StringUtils.equalsIgnoreCase(existing.getTaxonomy().getGenus(), updateA.getTaxonomy().getGenus()))
			;
		}).findFirst().orElse(null);
	}

	/**
	 * Check incoming JSON and update values accordingly
	 *
	 * @param update JSON node
	 * @param updateA Received accession data
	 * @param accession Genesys accession data
	 * @return
	 */
	private Accession applyChanges(JsonNode update, Accession updateA, Accession accession) {
		StopWatch stopWatch=StopWatch.createStarted();
		if (update.has("taxonomy")) {
			stopWatch.split();
			updateTaxonomy(update.get("taxonomy"), updateA.getTaxonomy(), accession);
			LOG.debug("Updated taxonomy for 1 accession in {}ms", stopWatch.getSplitTime());
		}
		update.fieldNames().forEachRemaining(fieldName -> {
			LOG.trace("Updating {}", fieldName);

			if ("doi".equals(fieldName)) {
				updateDoi(updateA.getDoi(), accession);
			
			} else if ("newInstituteCode".equals(fieldName)) {
				updateInstitute(update.get("newInstituteCode").textValue(), accession);
			
			} else if ("accessionNumber".equals(fieldName) || "newAcceNumb".equals(fieldName)) {
				updateAccessionNumber(update.get("accessionNumber"), update.get("newAcceNumb"), update.get("dataProviderId"), accession);

			} else if ("taxonomy".equals(fieldName)) {
//				stopWatch.split();
//				updateTaxonomy(update.get("taxonomy"), updateA.getTaxonomy(), accession);
//				LOG.debug("Updated taxonomy for 1 accession in {}ms", stopWatch.getSplitTime());

			} else if ("origCty".equals(fieldName)) {
				stopWatch.split();
				updateOrigCty(updateA.getOrigCty(), accession);
				LOG.debug("Updated origCty for 1 accession in {}ms", stopWatch.getSplitTime());

			} else if ("remarks".equals(fieldName)) {
				stopWatch.split();
				updateRemarks(update.get("remarks"), updateA.getAccessionId().getRemarks(), accession.getAccessionId());
				LOG.debug("Updated remarks for 1 accession in {}ms", stopWatch.getSplitTime());

			} else if ("coll".equals(fieldName)) {
				stopWatch.split();
				updateColl(update.get("coll"), updateA.getAccessionId().getColl(), accession.getAccessionId());
				LOG.debug("Updated coll for 1 accession in {}ms", stopWatch.getSplitTime());

			} else if ("acceName".equals(fieldName) || "otherNumb".equals(fieldName)) {
				stopWatch.split();
				updateAliases(fieldName, update.get(fieldName), accession.getAccessionId());
				LOG.debug("Updated aliases for 1 accession in {}ms", stopWatch.getSplitTime());

			} else if (Api1Constants.Accession.CROPNAME.equals(fieldName)) {
				// Noop
			} else if (Api1Constants.Accession.CROPCODE.equals(fieldName)) {
				// Noop
			} else if (ArrayUtils.contains(ACCESSIONID_FIELDS, fieldName)) {
				try {
					stopWatch.split();
					LOG.trace("Updating accessionId {}", fieldName, accession.getAccessionId());
					copy(AccessionId.class, updateA.getAccessionId(), accession.getAccessionId(), Iterators.forArray(fieldName));
					LOG.debug("Updated accessionId data for 1 accession in {}ms", stopWatch.getSplitTime());
				} catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
					throw new InvalidApiUsageException(e.getMessage(), e);
				}

			} else {
				try {
					stopWatch.split();
					copy(Accession.class, updateA, accession, Iterators.forArray(fieldName));
					LOG.debug("Updated the rest for 1 accession in {}ms", stopWatch.getSplitTime());
				} catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
					throw new InvalidApiUsageException(e.getMessage(), e);
				}
			}
		});

		processAccessionCrop(update, accession);

		// Fixes multiple donor alias entries
		updateDonorAlias(accession);
		return accession;
	}

	/**
	 * Allow owner to change accession's institute code.
	 * 
	 * @param newInstituteCode
	 * @param accession
	 */
	private void updateInstitute(String newInstituteCode, Accession accession) {
		if (! newInstituteCode.equalsIgnoreCase(accession.getInstituteCode())) {
			FaoInstitute newInstitute = faoInstituteRepository.findByCode(newInstituteCode);
			if (SecurityContextUtil.hasRole("ADMINISTRATOR") || SecurityContextUtil.hasPermission(newInstitute, "WRITE")) {
				accession.setInstitute(newInstitute);
				accessionCounter.recountInstitute(newInstitute);
			}
		}
	}

	private void updateDoi(String doi, Accession accession) {
		if (StringUtils.isNotBlank(doi)) {
			if (accession.getDoi() != null && ! StringUtils.equals(doi, accession.getDoi())) {
				throw new InvalidApiUsageException("Cannot update DOI from " + accession.getDoi() + " to " + doi);
			}
			accession.setDoi(doi);
		} else if (accession.getDoi() != null) {
			// Only reaches here if DOI is provided and blank
			throw new InvalidApiUsageException("Missing DOI for accession " + accession.getAccessionNumber() + " with doi=" + accession.getDoi());
		}
	}

	private void updateAccessionNumber(JsonNode nodeAcceNumb, JsonNode nodeNewNumb, JsonNode nodeDataProviderId, Accession accession) {
		String accessionNumber = nodeAcceNumb.textValue();
		String newAcceNumb = nodeNewNumb == null ? null : StringUtils.trimToNull(nodeNewNumb.textValue());
		String dataProviderId = nodeDataProviderId == null ? null : StringUtils.trimToNull(nodeDataProviderId.textValue());

		if (accession.isNew()) {
			if (newAcceNumb != null) {
				throw new InvalidApiUsageException("Refusing insert when NEWACCENUMB (newAcceNumb) is provided");
			}
			// Set provided accessionNumber and ignore newAcceNumb
			accession.setAccessionNumber(accessionNumber);

		} else if (dataProviderId != null && Objects.equals(dataProviderId, accession.getDataProviderId())) {
			// When accession has dataProviderId, update accession number
			accession.setAccessionNumber(accessionNumber);

		} else if (accession.getDoi() != null) {
			// DO NOT IMPLICITLY UPDATE ACCESSION NUMBERS
			if (! accession.getAccessionNumber().equalsIgnoreCase(accessionNumber)) {
				throw new InvalidApiUsageException("ACCENUMB for doi=" + accession.getDoi() + " does not match current " + accession.getAccessionNumber());
			}
		}

		if (StringUtils.isNotBlank(newAcceNumb)) {
			if (! StringUtils.equals(newAcceNumb, accession.getAccessionNumber())) {
				LOG.warn("Renaming accession {}: {} to {}", accession.getInstitute().getCode(), accession.getAccessionNumber(), newAcceNumb);
				accession.setAccessionNumber(newAcceNumb);
			}
		}
	}

	private void processAccessionCrop(JsonNode update, Accession accession) {
		// First process cropName
		JsonNode jsonNode = update.get(Api1Constants.Accession.CROPNAME);
		if (jsonNode == null) {
			// Do not make changes
		} else if (jsonNode.isNull()) {
			// Clear crop
			accession.setCropName(null);
			accession.setCrop(null);
		} else if (jsonNode.isTextual()) {
			// Update cropName
			accession.setCropName(StringUtils.abbreviate(jsonNode.asText(), 100));
			var autoCrop = cropService.getCrop(accession.getCropName());
			if (autoCrop != null) {
				// Update Crop if found by name, otherwise don't change
				accession.setCrop(autoCrop);
			}
		}

		// Then check for Genesys "cropCode"
		JsonNode cropNode = update.get(Api1Constants.Accession.CROPCODE);
		if (cropNode != null && cropNode.isTextual()) {
			// If provided find Crop by shortName
			Crop crop = cropService.getCrop(cropNode.asText());
			if (crop != null) {
				// Only if provided and found by Crop.shortName then override the value
				accession.setCrop(crop);
			}
		}
	}

	/**
	 * Keep only one DONORNUMB type alias
	 *
	 * @param accession the accession
	 */
	private void updateDonorAlias(Accession accession) {
		AccessionId accessionId = accession.getAccessionId();
		final List<AccessionAlias> allAliases = accessionId.getAliases() != null ? accessionId.getAliases() : new ArrayList<>();

		List<AccessionAlias> donorAliases = allAliases.stream().filter(al -> AliasType.DONORNUMB == al.getAliasType()).collect(Collectors.toList());
		AccessionAlias donorAlias = donorAliases.size() > 0 ? donorAliases.get(0) : null;

		if (donorAlias == null) {
			donorAlias = new AccessionAlias();
			donorAlias.setAliasType(AliasType.DONORNUMB);
			donorAlias.setAccession(accessionId);
			allAliases.add(donorAlias);
			donorAliases.add(donorAlias);
		}
		donorAlias.setUsedBy(accession.getDonorCode());
		donorAlias.setName(accession.getDonorNumb());

		for (int i = 0; i < donorAliases.size(); i++) {
			AccessionAlias alias = donorAliases.get(i);
			if (alias.isEmpty() || i > 0) {
				allAliases.remove(alias);
			}
		}
		accessionId.setAliases(allAliases);
	}

	private void updateAliases(String fieldName, JsonNode jsonNode, AccessionId accession) {
		if (jsonNode == null) {
			return;
		}

		final List<AccessionAlias> existing = accession.getAliases() != null ? accession.getAliases() : new ArrayList<>();

		AliasType aliasType = AccessionAlias.AliasType.valueOf(fieldName.toUpperCase());
		final List<AccessionAlias> newaliases = new ArrayList<>();

		if (jsonNode.isArray()) {
			ArrayNode ns = (ArrayNode) jsonNode;
			ns.forEach(n -> newaliases.add(validate(new AccessionAlias(aliasType, n.textValue()))));

		} else if (jsonNode.isTextual()) {
			newaliases.add(validate(new AccessionAlias(aliasType, jsonNode.textValue())));
		} else if (jsonNode.isNull()) {
			// NOOP
		} else {
			throw new InvalidApiUsageException("Don't know what to do with " + fieldName + "=" + jsonNode);
		}

		List<AccessionAlias> names = newaliases.stream().filter(alias -> !alias.isEmpty()).collect(Collectors.toList());

		List<AccessionAlias> toAdd = names.stream().filter(name -> existing.stream().filter(ex -> ex.equalTo(name)).count() == 0).collect(Collectors.toList());
		List<AccessionAlias> toRemove = existing.stream().filter(alias -> alias.getAliasType() == aliasType).filter(alias -> names.stream().filter(name -> alias.equalTo(name))
			.count() == 0).collect(Collectors.toList());

		if (!toRemove.isEmpty() || !toAdd.isEmpty()) {
			LOG.trace("Removing: {} Adding: {}", toRemove, toAdd);
		}
		existing.removeAll(toRemove);
		existing.addAll(toAdd);

		accession.setAliases(existing);
		existing.forEach(alias -> alias.setAccession(accession));
	}

	private void updateOrigCty(String origCty, Accession accession) {
		if (origCty == null) {
			accession.setCountryOfOrigin(null);

		} else if (accession.getCountryOfOrigin() == null || !accession.getCountryOfOrigin().getCode3().equals(origCty)) {
			Country country = geoService.getCountry(origCty);

			if (country == null) {
				// TODO
				// throw new InvalidApiUsageException("No country with ISO3 code " + origCty);
			}

			accession.setCountryOfOrigin(country);
		}
	}

	private void updateTaxonomy(JsonNode jsonNode, Taxonomy2 source, Accession accession) {
		Taxonomy2 taxonomy = accession.getTaxonomy();
		source.sanitize();

		if (jsonNode.has("newGenus")) {
			if (!jsonNode.has("species")) {
				throw new InvalidApiUsageException("Cannot specify newGenus without species.");
			}
			source.setGenus(jsonNode.get("newGenus").textValue());
		}

		if (taxonomy == null) {
			LOG.trace("Ensuring taxonomy {}", jsonNode);
			taxonomy = taxonomyService.ensureTaxonomy(source);
			accession.setTaxonomy(taxonomy);
		} else if (taxonomy.equalTo(source)) {
			// NOOP
		} else {
			LOG.trace("Updating taxonomy of {} with {}", accession.getAccessionNumber(), jsonNode);
			if (jsonNode.has(Api1Constants.Accession.GENUS) && jsonNode.has(Api1Constants.Accession.SPECIES)) {
				Taxonomy2 updated = new Taxonomy2(taxonomy);
				
				updated.setGenus(source.getGenus());
				updated.setSpecies(source.getSpecies());

				if (jsonNode.has(Api1Constants.Accession.SPAUTHOR)) {
					updated.setSpAuthor(source.getSpAuthor());
				} else {
					updated.setSpAuthor(null);
				}
				if (jsonNode.has(Api1Constants.Accession.SUBTAXA)) {
					updated.setSubtaxa(source.getSubtaxa());
					if (jsonNode.has(Api1Constants.Accession.SUBTAUTHOR)) {
						updated.setSubtAuthor(source.getSubtAuthor());
					} else {
						updated.setSubtAuthor(null);
					}
				} else {
					updated.setSubtaxa(null);
					updated.setSubtAuthor(null);
				}

				if (! updated.equalTo(taxonomy)) {
					LOG.trace("Ensuring taxonomy {}", updated);
					updated = taxonomyService.ensureTaxonomy(updated);
					accession.setTaxonomy(updated);
				}
			}
		}
	}

	private void updateColl(JsonNode jsonNode, AccessionCollect source, AccessionId accession) {
		AccessionCollect coll = accession.getColl();
		if (coll == null) {
			coll = new AccessionCollect();
			coll.setAccession(accession);
		}

		try {
			copy(AccessionCollect.class, source, coll, jsonNode.fieldNames());
			validate(coll);
		} catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException e) {
			throw new InvalidApiUsageException(e.getMessage(), e);
		}

		updateAliases("collNumb", jsonNode.get("collNumb"), accession);

		// removes empty
		accession.setColl(coll.isEmpty() ? null : coll);
	}

	private void updateRemarks(JsonNode jsonNode, List<AccessionRemark> remarks, AccessionId accession) {
		final List<AccessionRemark> existing = accession.getRemarks();
		if (existing != null) {
			if (remarks == null || remarks.isEmpty()) {
				existing.clear();
			} else {
				List<AccessionRemark> toAdd = remarks.stream().filter(rem -> rem != null && !rem.isEmpty()).filter(rem -> existing.stream().filter(ex -> rem.equalTo(ex))
					.count() == 0).collect(Collectors.toList());
				List<AccessionRemark> toRemove = existing.stream().filter(ex -> ex == null || ex.isEmpty() || remarks.stream().filter(rem -> ex.equalTo(rem)).count() == 0).collect(
					Collectors.toList());
				if (!toRemove.isEmpty() || !toAdd.isEmpty()) {
					LOG.trace("Removing: {} Adding: {}", toRemove, toAdd);
				}
				existing.removeAll(toRemove);
				existing.addAll(toAdd);
			}
		} else {
			accession.setRemarks(remarks);
		}

		// Make sure we take current remarks!
		if (accession.getRemarks() != null) {
			accession.getRemarks().stream().filter(remark -> remark != null).forEach(remark -> remark.setAccession(accession));
		}
	}

	@SuppressWarnings({ "unchecked" })
	private <T> void copy(Class<T> clazz, T source, T target, Iterator<String> fieldNames) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
		String fieldName = null;
		while (fieldNames.hasNext() && (fieldName = fieldNames.next()) != null) {
			LOG.trace("Copying {}.{}", clazz.getName(), fieldName);
			Field field = ReflectionUtils.findField(clazz, fieldName);
			if (field == null) {
				LOG.warn("Client provided field {} that does not exist on {}", fieldName, clazz);
				throw new InvalidApiUsageException("Field " + fieldName + " not valid for " + clazz.getName());
			}
			ReflectionUtils.makeAccessible(field);

			// Find getter for lazy-loading
			Method getter = findGetter(target.getClass(), field.getType(), fieldName);

			if (getter != null && field != null) {
				final Object srcValue = field.get(source);
				final Object dest = getter.invoke(target);
				// handle collections better
				LOG.trace("Original value for {} in {} is of type {}", fieldName, target.getClass(), dest == null ? "NULL" : dest.getClass());
				if (dest != null && dest instanceof Collection) {
					Collection<Object> collection = (Collection<Object>) dest;
					collection.clear();
					if (srcValue != null && srcValue instanceof Collection) {
						collection.addAll(((Collection<Object>) srcValue).stream().filter(o -> o != null).collect(Collectors.toList()));
						LOG.trace("Reset {} {} to {}", fieldName, dest.getClass(), srcValue);
					} else {
						LOG.trace("Cleared {} {}, source was empty", fieldName, dest.getClass());
					}
				} else {
					LOG.trace("Field {} is of type {}", fieldName, dest == null ? "NULL" : dest.getClass());
//					if (target instanceof Proxy || target instanceof HibernateProxy) {
						Method setter = findSetter(target.getClass(), field.getType(), fieldName);
						setter.invoke(target, srcValue);
						LOG.trace("Set {} to {}", fieldName, srcValue);
//					} else {
//						field.set(target, srcValue);
//						LOG.trace("Set {} to {}", fieldName, srcValue);
//					}
				}
			} else {
				LOG.warn("No getter|setter for field {}.{}", clazz.getName(), fieldName);
			}
		}
	}

	private Method findGetter(Class<?> clazz, Class<?> returnType, String fieldName) {
		String getterName = "get" + WordUtils.capitalize(fieldName);
		String getterName2 = "is" + WordUtils.capitalize(fieldName);

		LOG.trace("Looking for getter {}", getterName);
		for (Method method : clazz.getMethods()) {
			if (method.getParameterCount() == 0 && returnType.isAssignableFrom(method.getReturnType()) && (method.getName().equals(getterName) || method.getName().equals(getterName2))) {
				LOG.trace("Found getter {}", method);
				return method;
			}
		}

		throw new RuntimeException("No getter for field " + fieldName + " in " + clazz.getName());
	}

	private Method findSetter(Class<?> clazz, Class<?> parameterType, String fieldName) {
		String setterName = "set" + WordUtils.capitalize(fieldName);
		for (Method method : clazz.getMethods()) {
			if (method.getParameterCount() == 1 && method.getParameterTypes()[0].isAssignableFrom(parameterType) && method.getName().equals(setterName)) {
				LOG.trace("Found setter {}", method);
				return method;
			}
		}
		throw new RuntimeException("No setter for field " + fieldName + " in " + clazz.getName());
	}

	private <T extends BasicModel> T validate(T object) {
		Set<ConstraintViolation<T>> violations = validator.validate(object);
		if (violations != null && violations.size() > 0) {
			StringBuilder message = new StringBuilder("Validation failed: ");
			for (ConstraintViolation<T> cv : violations) {
				message.append(cv.getRootBeanClass().getSimpleName());
				message.append(".");
				message.append(cv.getPropertyPath().toString());
				message.append(" -> ");
				message.append(cv.getMessage());
				message.append("; ");
			}
			throw new ValidationException(message.toString());
		}
		return object;
	}

	@Transactional(timeout = 250, isolation = Isolation.READ_UNCOMMITTED, rollbackFor = Throwable.class)
	@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#institute, 'WRITE')")
	public List<AccessionOpResponse> deleteAccessions(FaoInstitute institute, ArrayNode identifiers) {
		assert (identifiers.isArray());
		StopWatch stopWatch = StopWatch.createStarted();
		ObjectReader reader = objectMapper.readerFor(Accession.class);

		final List<Accession> toBeDeleted = new ArrayList<>(identifiers.size());
		final List<AccessionOpResponse> responses = new ArrayList<>(identifiers.size());
		final List<Accession> toRemove = new ArrayList<>(identifiers.size());
		final List<AccessionHistoric> deleted = new ArrayList<>();


		for (JsonNode accn : identifiers) {
			LOG.trace("Received: {}", accn);
			try {
				Accession accession = reader.readValue(accn);
				var jsonUuid = accn.get("uuid");
				if (jsonUuid != null && jsonUuid.isTextual()) accession.getAccessionId().setUuid(UUID.fromString(jsonUuid.textValue()));

				if (accession.getInstituteCode() == null || (accession.getDoi() == null && accession.getAccessionNumber() == null)) {
					throw new InvalidApiUsageException("instituteCode, accessionNumber OR doi missing in " + accn);
				}

				// Sanity check
				if (!accession.getInstituteCode().equals(institute.getCode())) {
					throw new InvalidApiUsageException("Accession does not belong to institute " + institute.getCode());
				}

				toBeDeleted.add(accession);
			} catch (IOException e) {
				LOG.error("Could not parse input: {}", e.getMessage(), e);
				toBeDeleted.add(null); // need to match get(index) with responses and updates

				AccessionOpResponse response = new AccessionOpResponse(accn);
				response.setResult(new UpsertResult(UpsertResult.Type.ERROR)).setError(e.getMessage());
				responses.add(response);
			}
		}
		LOG.debug("Processed incoming JSON for {} accessions in {}ms", identifiers.size(), stopWatch.getTime());

		List<Accession> existingAccessions = accessionRepository.find(! institute.hasUniqueAcceNumbs(), toBeDeleted);
		LOG.debug("Have {} accessions for removal and {} exist in {}ms", toBeDeleted.size(), existingAccessions.size(), stopWatch.getTime());

		for (Accession deletion: toBeDeleted) {
			AccessionOpResponse response;
			responses.add(response = new AccessionOpResponse(deletion));

			Accession accession = findExistingAccession(institute, existingAccessions, deletion);

			if (accession == null) {
				response.setResult(new UpsertResult(UpsertResult.Type.NOOP));
				response.setError("Record not found");
			} else {
				try {
					if (accession.getDoi() != null) {
						response.setResult(new UpsertResult(UpsertResult.Type.ERROR));
						response.setError("Accessions with a DOI cannot be deleted");
						continue;
					}
					if (! accession.getInstitute().getId().equals(institute.getId())) {
						response.setResult(new UpsertResult(UpsertResult.Type.ERROR));
						response.setError("INSTCODE does not match");
						continue;
					}
					if (deletion.getUuid() != null && ! Objects.equals(deletion.getUuid(), accession.getUuid())) {
						response.setResult(new UpsertResult(UpsertResult.Type.ERROR));
						response.setError("Accession UUID does not match");
						continue;
					}

					LOG.trace("Deleting accession {}", accession);
					toRemove.add(accession);
					response.setResult(new UpsertResult(UpsertResult.Type.DELETE).setUUID(accession.getUuid()));

					AccessionHistoric hist = new AccessionHistoric();
					SelfCopy.copy(accession, hist);
					hist.setAccessionId(accessionIdRepository.findById(accession.getId()).orElse(null));
					deleted.add(hist);
				} catch (InvalidApiUsageException e) {

					if (LOG.isDebugEnabled()) {
						LOG.error(e.getMessage(), e);
					} else {
						LOG.error(e.getMessage());
					}

					response.setError(e.getMessage());
				}
			}
		}

		if (!toRemove.isEmpty()) {
			accessionRepository.deleteAll(toRemove);
			LOG.trace("Done deleting: {},  now adding", toRemove);
			accessionHistoricRepository.saveAll(deleted);
		}

		accessionCounter.recountInstitute(institute);

		return responses;
	}

}