AccessionController.java

/**
 * Copyright 2014 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **/

package org.genesys.server.api.v0;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import javax.persistence.PersistenceException;
import javax.persistence.RollbackException;
import javax.validation.Valid;
import javax.validation.ValidationException;

import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.PleaseRetryException;
import org.genesys.server.api.model.AccessionHeaderJson;
import org.genesys.server.api.model.Api0Constants;
import org.genesys.server.api.model.Api1Constants;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.exception.NotFoundElement;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.GeoService;
import org.genesys.server.service.InstituteService;
import org.genesys.server.service.TaxonomyService;
import org.genesys.server.exception.NonUniqueAccessionException;
import org.genesys.server.service.impl.RESTApiException;
import org.genesys.server.service.impl.RESTApiValueException;
import org.genesys.server.exception.SearchException;
import org.genesys.server.service.worker.AccessionOpResponse;
import org.genesys.server.service.worker.AccessionUploader;
import org.genesys.server.service.worker.AccessionOpResponse.UpsertResult;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.CannotAcquireLockException;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.TransactionException;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

import io.swagger.annotations.Api;

@RestController("accessionApi0")
@PreAuthorize("isAuthenticated()")
@RequestMapping(value = { AccessionController.CONTROLLER_URL, "/json/v0/acn" })
@Api(tags = { "accession" })
public class AccessionController extends ApiBaseController {
	private static final int UPLOAD_RETRIES = 2;
	public static final String CONTROLLER_URL = ApiBaseController.APIv0_BASE + "/acn";

	private final ObjectMapper mapper = new ObjectMapper();

	@Autowired
	GenesysService genesysService;

	@Autowired
	private AccessionUploader uploader;

	@Autowired
	InstituteService instituteService;

	@Autowired
	GeoService geoService;

	@Autowired
	TaxonomyService taxonomyService;

	@Autowired(required = false)
	ElasticsearchService elasticService;

	/**
	 * Check if accessions exists in the system
	 * 
	 * @return
	 * @throws NonUniqueAccessionException
	 */
	@RequestMapping(value = "/exists/{instCode}/{genus:.+}", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE })
	public @ResponseBody boolean exists(@PathVariable("instCode") String instCode, @PathVariable("genus") String genus, @RequestParam("acceNumb") String acceNumb)
			throws NonUniqueAccessionException {
		if (LOG.isDebugEnabled()) {
			LOG.debug("Checking if accn exists {}.{} genus={}", instCode, acceNumb, genus);
		}

		final String doi = null;
		final Accession accession = genesysService.getAccession(instCode, doi, acceNumb, genus);

		if (accession == null) {
			LOG.warn("No accession {}.{} genus={}", instCode, acceNumb, genus);
		}

		return accession != null;
	}

	/**
	 * Check if accessions exists in the system
	 * 
	 * @return
	 * @throws IOException
	 * @throws JsonProcessingException
	 * @throws RESTApiValueException
	 * @throws NonUniqueAccessionException
	 */
	@RequestMapping(value = "/{instCode}/check", method = { RequestMethod.POST, RequestMethod.PUT }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public @ResponseBody JsonNode check(@PathVariable("instCode") String instCode, @RequestBody JsonNode json) throws JsonProcessingException, IOException, RESTApiValueException,
			NonUniqueAccessionException {
		final FaoInstitute institute = instituteService.getInstitute(instCode);
		if (institute == null) {
			throw new NotFoundElement("No institute " + instCode);
		}

		final List<String> batch = new ArrayList<String>();

		if (json.isArray()) {
			for (final JsonNode j : json) {
				if (j.isNull() || !j.isTextual()) {
					throw new RESTApiValueException("acceNumb must be a non-null String");
				}
				batch.add(j.textValue());
			}
		} else {
			if (json.isNull() || !json.isTextual()) {
				throw new RESTApiValueException("acceNumb must be a non-null String");
			}
			batch.add(json.textValue());
		}

		LOG.info("Batch processing {} entries", batch.size());
		final ArrayNode rets = mapper.createArrayNode();

		for (final String acceNumb : batch) {
			if (LOG.isDebugEnabled()) {
				LOG.debug("Loading accession {} from {}", acceNumb, instCode);
			}

			if (institute.hasUniqueAcceNumbs()) {
				final Accession accession = genesysService.getAccession(instCode, acceNumb);
				rets.add(accession == null ? null : accession.getAccessionId().getId());
			} else {
				final List<Accession> accessions = genesysService.listAccessions(institute, acceNumb);
				if (accessions.size() == 1) {
					rets.add(accessions.get(0).getAccessionId().getId());
				} else {
					final ArrayNode ret = rets.arrayNode();
					for (final Accession accession : accessions) {
						ret.add(accession.getAccessionId().getId());
					}
					rets.add(ret);
				}
			}
		}

		return rets;
	}

	/**
	 * Update accessions in the system
	 * @return 
	 * 
	 * @return
	 * @throws IOException
	 * @throws JsonProcessingException
	 * @throws RESTApiException
	 */
	@RequestMapping(value = "/{instCode}/upsert", method = { RequestMethod.POST, RequestMethod.PUT }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public List<AccessionOpResponse> upsertInstituteAccession(@PathVariable("instCode") String instCode, @RequestBody ArrayNode updates) throws Exception {
		
		StopWatch stopWatch = StopWatch.createStarted();

		LOG.trace("Received:\n {}", updates);
		upgradeToV2(updates);

		// User's permission to WRITE to this WIEWS institute are checked in
		// BatchRESTService.
		final FaoInstitute institute = instituteService.getInstitute(instCode);
		if (institute == null) {
			throw new NotFoundElement();
		}


		for (int tryCount = 0; tryCount < UPLOAD_RETRIES; tryCount++) {
			try {
				List<AccessionOpResponse> res = uploader.upsertAccessions(institute, updates);
				LOG.info("Processed {} accessions for {} in {} tries in {}ms", updates.size(), instCode, tryCount + 1, stopWatch.getTime());
				return res;
			} catch (PleaseRetryException e) {
				LOG.error("Retry {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
				// Wait a bit
				Thread.sleep((long) (RandomUtils.nextDouble(10, 100) * tryCount));
			} catch (Throwable e) {
				if (e instanceof ValidationException || e instanceof DataIntegrityViolationException) {

				} else {
					LOG.warn("Error inserting bulk: {}", e.getMessage(), e);
				}
				List<AccessionOpResponse> res = upsert1By1(institute, updates);
				LOG.info("Processed {} accessions for {} 1by1 after {} tries in {}ms because of {}", updates.size(), instCode, tryCount + 1, stopWatch.getTime(), e.getMessage());
				return res;
			}
		}
		throw new InvalidApiUsageException("Just couldn't do it.");
	}

	private List<AccessionOpResponse> upsert1By1(FaoInstitute institute, ArrayNode updates) throws Exception {
		List<AccessionOpResponse> response = new ArrayList<>();
		ArrayNode single = mapper.createArrayNode();
		for (JsonNode update : updates) {
			single.removeAll();
			single.add(update);
			Throwable lastException = null;
			for (int tryCount = 0; tryCount < UPLOAD_RETRIES; tryCount++) {
				try {
					response.addAll(uploader.upsertAccessions(institute, single));
					lastException = null;
					break;
				} catch (ValidationException | InvalidApiUsageException e) {
					lastException = e;
					break; // don't retry
				} catch (TransactionException | RollbackException | CannotAcquireLockException | DataIntegrityViolationException | PleaseRetryException e) {
					lastException = e;
					if (tryCount > 1) {
						LOG.info("Retry {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
					} else {
						LOG.debug("Retrying {} of {} tries due to: {}", tryCount, UPLOAD_RETRIES, e.getMessage());
					}
					// Wait a bit
					Thread.sleep((long) (RandomUtils.nextDouble(10, 100) * tryCount));
				}
			}
			if (lastException != null) {
				if (LOG.isDebugEnabled()) {
					LOG.debug("Upsert failed due to: {} data={}", lastException.getMessage(), update, lastException);
				} else {
					LOG.info("Upsert failed due to: {} data={}", lastException.getMessage(), update);
				}
				// record error
				response.add(new AccessionOpResponse(update)
					.setResult(new UpsertResult(UpsertResult.Type.ERROR)).setError(getDetailedErrorMessage(lastException)));
			}
		}
		return response;
	}

	private String getDetailedErrorMessage(Throwable e) {
		StringBuffer sb=new StringBuffer(e.getMessage());
		Throwable r = e;
		while ((r = r.getCause()) !=null) {
			sb.append("\n" + r.getMessage());
		}	
		return sb.toString();
	}

	/**
	 * Delete accessions by acceNumb (and genus)?
	 * 
	 * @return
	 * @throws RESTApiException
	 * @throws IOException
	 * @throws JsonProcessingException
	 */
	@RequestMapping(value = "/{instCode}/delete-named", method = { RequestMethod.POST, RequestMethod.PUT }, consumes = { MediaType.APPLICATION_JSON_VALUE }, produces = {
			MediaType.APPLICATION_JSON_VALUE })
	public @ResponseBody List<AccessionOpResponse> deleteAccessions(@PathVariable("instCode") String instCode, @RequestBody @Valid List<AccessionHeaderJson> batch) throws RESTApiException {
		// User's permission to WRITE to this WIEWS institute are checked in
		// BatchRESTService.
		final FaoInstitute institute = instituteService.getInstitute(instCode);
		if (institute == null) {
			throw new NotFoundElement();
		}

		final ArrayNode updates = convertToArrayNode(batch);
		try {
			List<AccessionOpResponse> response = null;
			try {
				response = uploader.deleteAccessions(institute, updates);
				LOG.info("Deleted {} accessions from {}", response.size(), instCode);
			} catch (DataAccessException | TransactionException | PersistenceException e) {
				LOG.info("Retrying delete one by one due to {}", e.getMessage());
				response = deleteAccessions1by1(institute, updates);
			}
			return response;

		} catch (CannotAcquireLockException e) {
			throw new PleaseRetryException("Operation failed, please retry.", e);
		}
	}

	private List<AccessionOpResponse> deleteAccessions1by1(FaoInstitute institute, ArrayNode updates) {
		LOG.info("Attempting delete 1 by 1");

		ArrayNode single = mapper.createArrayNode();
		List<AccessionOpResponse> response = new ArrayList<AccessionOpResponse>();
		for (JsonNode update : updates) {
			try {
				single.removeAll();
				single.add(update);
				AccessionOpResponse accessionResponse = uploader.deleteAccessions(institute, single).get(0);
				response.add(accessionResponse);

			} catch (Throwable e) {
				if (LOG.isInfoEnabled()) {
					LOG.info("Error deleting {}: {}", update.get("instituteCode").asText(), e.getMessage());
				}

				AccessionOpResponse accessionResponse = new AccessionOpResponse(update);
				accessionResponse.setError(getDetailedErrorMessage(e));
				response.add(accessionResponse);
			}
		}
		return response;
	}

	@RequestMapping(value = "/search", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public @ResponseBody List<Accession> search(@RequestParam("page") final int page, @RequestParam("query") final String query) throws SearchException {
		return elasticService.search(null, StringUtils.defaultIfBlank(query, "*"), Accession.class); //, PageRequest.of(page - 1, 20));
	}

	// FIXME Not using institute...
	@RequestMapping(value = "/{instCode}/list", method = { RequestMethod.GET }, produces = { MediaType.APPLICATION_JSON_VALUE })
	public @ResponseBody List<Accession> list(@PathVariable("instCode") String instCode, @RequestParam("page") int page, @RequestParam("query") String query)
			throws SearchException {
		FaoInstitute institute = instituteService.getInstitute(instCode);
		if (institute == null) {
			throw new NotFoundElement();
		}
		query = StringUtils.defaultIfBlank(query, "*");
		// TODO filer by inst
		return elasticService.search(null, query, Accession.class); // , PageRequest.of(page - 1, 20));
	}

	private ArrayNode convertToArrayNode(List<AccessionHeaderJson> updates) {
		final ArrayNode arrayNode = mapper.createArrayNode();
		for (AccessionHeaderJson accnJ: updates) {
			ObjectNode accession = arrayNode.addObject();
			accession.put("instituteCode", accnJ.getHoldingInstitute());
			accession.put("accessionNumber", accnJ.getAccessionNumber());
			accession.put("doi", accnJ.getDoi());
			accession.put("dataProviderId", accnJ.getDataProviderId());
			if (accnJ.getUuid() != null) accession.put("uuid", accnJ.getUuid().toString());

			ObjectNode taxonNode = accession.putObject("taxonomy");
			taxonNode.put("genus", accnJ.getGenus());
		}
		return arrayNode;
	}

	private void upgradeToV2(ArrayNode updates) {
		for (JsonNode update : updates) {
			upgradeToV2((ObjectNode) update);
		}
	}

	private void upgradeToV2(ObjectNode update) {
		upgradeRename(update, Api0Constants.Accession.INSTCODE, Api1Constants.Accession.INSTCODE);
		upgradeRename(update, Api0Constants.Accession.ACCENUMB, Api1Constants.Accession.ACCENUMB);
		upgradeRename(update, Api0Constants.Accession.SPAUTHOR, Api1Constants.Accession.SPAUTHOR);
		upgradeRename(update, Api0Constants.Accession.SUBTAUTHOR, Api1Constants.Accession.SUBTAUTHOR);
		upgradeRename(update, Api0Constants.Accession.ACQDATE, Api1Constants.Accession.ACQDATE);
		upgradeRename(update, Api0Constants.Accession.BREDCODE, Api1Constants.Accession.BREDCODE);
		upgradeRename(update, Api0Constants.Accession.BREDNAME, Api1Constants.Accession.BREDNAME);
		upgradeRename(update, Api0Constants.Accession.MLSSTAT, Api1Constants.Accession.MLSSTAT);
		upgradeRename(update, Api0Constants.Accession.ORIGCTY, Api1Constants.Accession.ORIGCTY);

		upgradeToArray(update, Api1Constants.Accession.BREDCODE); // renamed above
		upgradeToArray(update, Api1Constants.Accession.BREDNAME); // renamed above
		
		if (update.has("geo")) {
			ObjectNode geo = (ObjectNode) update.get("geo");
			upgradeRename(geo, Api0Constants.Geo.COORDUNCERT, Api1Constants.Geo.COORDUNCERT);
			upgradeRename(geo, Api0Constants.Geo.COORDDATUM, Api1Constants.Geo.COORDDATUM);
			upgradeRename(geo, Api0Constants.Geo.GEOREFMETH, Api1Constants.Geo.GEOREFMETH);
		}

		if (update.has("coll")) {
			ObjectNode coll = (ObjectNode) update.get("coll");
			upgradeToArray(coll, Api1Constants.Collecting.COLLCODE);
			upgradeToArray(coll, Api1Constants.Collecting.COLLNAME);
			upgradeToArray(coll, Api1Constants.Collecting.COLLINSTADDRESS);
		}
		
		ObjectNode taxonomy;
		if (update.has("taxonomy")) {
			taxonomy = (ObjectNode) update.get("taxonomy");
		} else {
			taxonomy = update.putObject("taxonomy");
		}
		
		upgradeTaxonomy(update, taxonomy, Api1Constants.Accession.GENUS);
		upgradeTaxonomy(update, taxonomy, Api1Constants.Accession.SPECIES);
		upgradeTaxonomy(update, taxonomy, Api1Constants.Accession.SPAUTHOR);
		upgradeTaxonomy(update, taxonomy, Api1Constants.Accession.SUBTAXA);
		upgradeTaxonomy(update, taxonomy, Api1Constants.Accession.SUBTAUTHOR);
	}

	private void upgradeToArray(ObjectNode obj, String fieldName) {
		JsonNode node = obj.get(fieldName);
		if (node != null) {
			if (! node.isNull() && ! node.isArray()) {
				ArrayNode arrNode = obj.putArray(fieldName);
				for (String x : node.asText().split(";")) {
					if (StringUtils.isNotBlank(x)) {
						arrNode.add(StringUtils.trim(x));
					}
				}
			}
		}
	}

	private void upgradeRename(ObjectNode update, String v1name, String v2name) {
		if (update.has(v1name)) {
			update.set(v2name, update.remove(v1name));
		}
	}

	private void upgradeTaxonomy(ObjectNode source, ObjectNode taxonomy, String fieldName) {
		if (source.has(fieldName)) {
			if (taxonomy.has(fieldName)) {
				source.remove(fieldName);
			} else {
				taxonomy.set(fieldName, source.remove(fieldName));
			}
		}
	}
}