AdminController.java

/*
 * Copyright 2022 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.genesys.server.api.admin.v1;

import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;

import javax.validation.constraints.NotNull;

import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.oauth.model.Authorization;
import org.genesys.blocks.oauth.model.QAuthorization;
import org.genesys.blocks.oauth.persistence.AuthorizationRepository;
import org.genesys.blocks.oauth.service.OAuthClientService;
import org.genesys.blocks.security.NoUserFoundException;
import org.genesys.blocks.security.service.CustomAclService;
import org.genesys.filerepository.InvalidRepositoryPathException;
import org.genesys.filerepository.model.ImageGallery;
import org.genesys.filerepository.model.RepositoryFolder;
import org.genesys.filerepository.persistence.ImageGalleryPersistence;
import org.genesys.filerepository.persistence.RepositoryFilePersistence;
import org.genesys.filerepository.persistence.RepositoryFolderRepository;
import org.genesys.filerepository.service.ImageGalleryService;
import org.genesys.filerepository.service.RepositoryService;
import org.genesys.server.api.ApiBaseController;
import org.genesys.server.api.Pagination;
import org.genesys.server.exception.InvalidApiUsageException;
import org.genesys.server.model.dataset.Dataset;
import org.genesys.server.model.genesys.Accession;
import org.genesys.server.model.genesys.AccessionId;
import org.genesys.server.model.genesys.PDCI;
import org.genesys.server.model.genesys.QAccessionId;
import org.genesys.server.model.impl.FaoInstitute;
import org.genesys.server.persistence.AccessionIdRepository;
import org.genesys.server.persistence.AccessionRepository;
import org.genesys.server.persistence.FaoInstituteRepository;
import org.genesys.server.persistence.PDCIRepository;
import org.genesys.server.persistence.SubsetRepository;
import org.genesys.server.persistence.dataset.DatasetRepository;
import org.genesys.server.persistence.kpi.ExecutionRepository;
import org.genesys.server.service.AccessionService;
import org.genesys.server.service.AdminService;
import org.genesys.server.service.ArticleService;
import org.genesys.server.service.ContentService;
import org.genesys.server.service.CountryNamesUpdater;
import org.genesys.server.service.DatasetService;
import org.genesys.server.service.ElasticsearchService;
import org.genesys.server.service.GenesysService;
import org.genesys.server.service.GeoRegionService;
import org.genesys.server.service.GeoService;
import org.genesys.server.service.InstituteService;
import org.genesys.server.service.TaxonomyService;
import org.genesys.server.service.UserService;
import org.genesys.server.service.filter.AccessionFilter;
import org.genesys.server.exception.NonUniqueAccessionException;
import org.genesys.server.service.worker.AccessionCounter;
import org.genesys.server.service.worker.AccessionProcessor;
import org.genesys.server.service.worker.ITPGRFAStatusUpdater;
import org.genesys.server.service.worker.InstituteUpdater;
import org.genesys.server.service.worker.SGSVUpdate;
import org.genesys.server.service.worker.ScheduledGLISUpdater;
import org.genesys.server.service.worker.Taxonomy2GRINMatcher;
import org.genesys.server.service.worker.UsdaTaxonomyUpdater;
import org.genesys.server.service.worker.WorldClimUpdater;
import org.genesys.util.PDCICalculator;
import org.genesys.util.TileIndexCalculator;
import org.genesys.worldclim.WorldClimUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.nimbusds.oauth2.sdk.GrantType;
import com.querydsl.jpa.impl.JPAQueryFactory;

import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.Parameter;

/**
 * @author Maxym Borodenko
 */
@RestController("adminApi1")
@PreAuthorize("hasRole('ADMINISTRATOR')")
@RequestMapping(AdminController.CONTROLLER_URL)
@Api(tags = { "adminv1" })
public class AdminController extends ApiBaseController{

	/** The Constant CONTROLLER_URL. */
	public static final String CONTROLLER_URL = APIv1_ADMIN_BASE;

	public static final String ELASTIC_SEARCH_URL = "/elastic";

	/** The Constant LOG. */
	public static final Logger LOG = LoggerFactory.getLogger(AdminController.class);

	@Autowired
	private RepositoryService repositoryService;

	@Autowired
	private FaoInstituteRepository instituteRepository;

	@Autowired
	private DatasetRepository datasetRepository;

	@Autowired
	private ImageGalleryService imageGalleryService;

	@Autowired
	private CustomAclService aclService;

	@Autowired
	private DatasetService datasetService;

	@Autowired
	private ScheduledGLISUpdater scheduledGLISUpdater;

	@Autowired
	private TaskExecutor taskExecutor;

	@Autowired(required = false)
	private ElasticsearchService elasticsearchService;

	@Autowired
	InstituteUpdater instituteUpdater;

	@Autowired
	GeoService geoService;

	@Autowired
	private InstituteService instituteService;

	@Autowired
	GenesysService genesysService;

	@Autowired
	SGSVUpdate sgsvUpdater;

	@Autowired
	private AccessionService accessionService;

	@Autowired
	CountryNamesUpdater alternateNamesUpdater;

	@Autowired
	ITPGRFAStatusUpdater itpgrfaUpdater;

	@Autowired
	private UsdaTaxonomyUpdater usdaTaxonomyUpdater;

	@Autowired
	private Taxonomy2GRINMatcher genesysTaxonomy2GRIN;

	@Autowired
	private AccessionCounter accessionCounter;

	@Autowired
	private AccessionProcessor accessionProcessor;

	@Autowired
	private PDCIRepository pdciRepository;

	@Autowired
	private JPAQueryFactory jpaQueryFactory;

	@Autowired
	private AccessionIdRepository accessionIdRepository;

	@Autowired
	private ExecutionRepository kpiExecutionRepository;

	@Autowired
	private GeoRegionService geoRegionService;

	@Autowired
	private AccessionRepository accessionRepository;

	@Autowired
	private TaxonomyService taxonomyService;

	@Autowired
	private WorldClimUpdater worldClimUpdater;

	@Autowired
	private AdminService adminService;

	@Autowired
	private UserService userService;

	@Autowired
	private OAuthClientService oAuthClientService;

	@Autowired
	private AuthorizationRepository authorizationRepository;
	
	@Autowired
	private ArticleService articleService;

	@GetMapping(value = "/client/{clientId}/auth")
	@Transactional(readOnly = true)
	public Page<Authorization> findClientAuthorizations(@PathVariable String clientId, @Parameter(hidden = true) final Pagination page) {
		var client = oAuthClientService.getClient(clientId);
		if (client == null) {
			throw new InvalidApiUsageException("Client not found");
		}
		Pageable pageable = ArrayUtils.isEmpty(page.getS()) ? page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC) : page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE);
		return authorizationRepository.findAll(QAuthorization.authorization.registeredClientId.eq(client.getId())
			.and(QAuthorization.authorization.authorizationGrantType.eq(GrantType.CLIENT_CREDENTIALS.getValue())), pageable);
	}

	@GetMapping(value = "/user/{uuid}/auth")
	@Transactional(readOnly = true)
	public Page<Authorization> findUserAuthorizations(@PathVariable("uuid") final UUID uuid, @Parameter(hidden = true) final Pagination page) throws NoUserFoundException {
		var user = userService.getUser(uuid);
		if (user == null) {
			throw new NoUserFoundException();
		}
		Pageable pageable = ArrayUtils.isEmpty(page.getS()) ? page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE, Sort.Direction.ASC) : page.toPageRequest(MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE);
		return authorizationRepository.findAll(QAuthorization.authorization.principalName.eq(user.getUsername())
			.and(QAuthorization.authorization.authorizationGrantType.eq(GrantType.AUTHORIZATION_CODE.getValue())), pageable);
	}

	@DeleteMapping(value = "/auth", produces = { MediaType.APPLICATION_JSON_VALUE })
	public void removeAuthorizations(@RequestBody final List<String> authIds) {
		var authorizations = authorizationRepository.findAll(QAuthorization.authorization.id.in(authIds));
		authorizationRepository.deleteAll(authorizations);
	}

	@PostMapping(value = "/ensure-inst-folders")
	public Boolean ensureInstituteFolders() throws Exception {
		LOG.info("Ensure institute folders");
		List<RepositoryFolder> wiewsFolders = repositoryService.getFolders(Paths.get("/wiews"), RepositoryFolder.DEFAULT_SORT);
		for (RepositoryFolder instFolder: wiewsFolders) {
			LOG.warn("Ensuring inheritance for {}", instFolder.getFolderPath());
			FaoInstitute institute = instituteRepository.findByCode(instFolder.getName());
			if (institute != null) {
				repositoryService.ensureFolder(instFolder.getFolderPath(), institute);
			} else {
				LOG.error("No institute " + instFolder.getName() + " for path " + instFolder.getPath());
			}
		}
		return true;
	}

	@PostMapping(value = "/ensure-dataset-folders")
	public Boolean ensureDatasetFolders() throws Exception {
		LOG.info("Ensure dataset folders");
		for (Dataset dataset : datasetRepository.findAll()) {
			final Path datasetPath = datasetService.getDatasetRepositoryFolder(dataset);
			LOG.warn("Ensuring inheritance for {}", datasetPath);
			repositoryService.ensureFolder(datasetPath, dataset.getOwner());
		}
		return true;
	}

	/**
	 * We want all thumbnails to exist
	 */
	@PostMapping(value = "/ensure-thumbnails")
	public Boolean ensureThumbnails() {
		LOG.info("Ensure thumbnails");
		int page = 0;
		Page<ImageGallery> galleries = null;
		do {
			PageRequest p = PageRequest.of(page++, 10);
			galleries = imageGalleryService.listImageGalleries(p);
			galleries.forEach(gallery -> {
				LOG.warn("Ensuring all thumbnails for {}", gallery.getPath());
				imageGalleryService.ensureThumbnails(gallery);
			});

		} while (galleries.hasNext());
		return true;
	}

	/**
	 * This method refreshes data in the currently active index. It is very handy
	 * when having to refresh part of ES after direct database update.
	 *
	 * @param type
	 */
	@PostMapping(value = ELASTIC_SEARCH_URL + "/reindex/{type}")
	public Boolean reindexElasticContent(@PathVariable(required = true) String type) {
		if (type.equals("All")) {
			taskExecutor.execute(() -> {
				try {
					elasticsearchService.reindexAll();
				} catch (Throwable e) {
					LOG.error("Error executing reindexAll", e);
				}
			});
		} else {
			taskExecutor.execute(() -> {
				try {
					elasticsearchService.reindex(Class.forName(type));
				} catch (Throwable e) {
					LOG.error("Error executing reindex of " + type, e);
				}
			});
		}
		return true;
	}

	/**
	 * This method refreshes data in the currently active index. It is very handy
	 * when having to refresh part of ES after direct database update.
	 *
	 * @param jsonFilter
	 * @throws IOException
	 */
	@PostMapping(value = ELASTIC_SEARCH_URL + "/reindex")
	public Boolean reindexElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter) throws IOException {
		AccessionFilter accessionFilter = new ObjectMapper().registerModule(new JavaTimeModule()).readValue(jsonFilter, AccessionFilter.class);
		taskExecutor.execute(() -> {
			try {
				elasticsearchService.reindex(Accession.class, accessionFilter);
			} catch (Throwable e) {
				LOG.error("Error executing reindex Accession", e);
			}
		});
		return true;
	}

	/**
	 * This method removes data in the currently active index. It is very handy
	 * when having to refresh part of ES after direct database update.
	 *
	 * @param jsonFilter
	 * @throws IOException
	 */
	@PostMapping(value = ELASTIC_SEARCH_URL + "/remove")
	public Boolean removeElasticFiltered(@RequestParam(value = "filter", required = true) String jsonFilter) throws IOException {
		AccessionFilter accessionFilter = new ObjectMapper().registerModule(new JavaTimeModule()).readValue(jsonFilter, AccessionFilter.class);
		taskExecutor.execute(() -> {
			try {
				elasticsearchService.remove(Accession.class, accessionFilter);
			} catch (Throwable e) {
				LOG.error("Error executing reindex Accession", e);
			}
		});
		return true;
	}

	@PostMapping(value = "/cleanup-acl")
	public Boolean cleanupAcl() {
		LOG.info("Cleanup ACL");
		aclService.cleanupAcl();
		return true;
	}

	@PostMapping(value = "/update-glis")
	public Boolean updateGLIS(@RequestParam(value="from", required = true) @DateTimeFormat(pattern="yyyy-MM-dd") final LocalDate from) {
		LOG.info("Update GLIS with accessions with DOI");
		scheduledGLISUpdater.notifyGLIS(from.atStartOfDay().toInstant(ZoneOffset.UTC));
		return true;
	}

	@PostMapping(value = "/institute/update")
	public void refreshWiews() throws Exception {
		instituteUpdater.updateFaoInstitutes();
	}

	@PostMapping(value = "/geo/refreshCountries")
	public void refreshCountries() throws Exception {
		geoService.updateCountryData();
	}

	@PostMapping(value = "/updateAccessionCountryRefs")
	public void updateAccessionCountryRefs() {
		genesysService.updateAccessionCountryRefs();
	}

	@PostMapping(value = "/updateInstituteCountryRefs")
	public void updateInstituteCountryRefs() {
		instituteService.updateCountryRefs();
	}

	@PostMapping(value = "/updateAccessionInstituteRefs")
	public void updateAccessionInstituteRefs() {
		genesysService.updateAccessionInstitueRefs();
	}

	@PostMapping(value = "/scanForSubsets")
	public void scanForSubsets() {
		accessionService.scanForPublishedSubsets();
	}

	@PostMapping(value = "/scanForDatasets")
	public void scanForDatasets() {
		accessionService.scanForPublishedDatasets();
	}

	@PostMapping(value = "/resetCounters")
	public void resetCounters() {
		accessionService.resetSubsetAndDatasetCounters();
	}

	@PostMapping(value = "/dataset/relinkDatasetAccessions")
	public void rematchDatasetAccessions() {
		datasetService.rematchDatasetAccessions();
	}

	@PostMapping(value = "/updateSGSV")
	public void updateSGSV() {
		sgsvUpdater.updateSGSV();
	}

	@PostMapping(value = "/content/sanitize-html")
	public void sanitize() {
		LOG.info("Sanitizing content");
		articleService.sanitizeAll();
		LOG.info("Sanitizing content.. Done");
	}

	@PostMapping(value = "/geo/update-alt-names")
	public void updateAlternateNames() throws Exception {
		LOG.info("Updating alternate GEO names");
		alternateNamesUpdater.updateAlternateNames();
		LOG.info("Updating alternate GEO names: done");
	}

	@PostMapping(value = "/updateITPGRFA")
	public void updateITPGRFA() throws Exception {
		LOG.info("Updating country ITPGRFA status");
		itpgrfaUpdater.downloadAndUpdate();
		LOG.info("Updating done");
	}

	@PostMapping(value = "/taxonomy/update-grin")
	public void updateGRIN() throws Exception {
		LOG.info("Updating GRIN Taxonomy");
		usdaTaxonomyUpdater.update();
		LOG.info("Updating done");
	}

	@PostMapping(value = "/taxonomy/map-to-grin")
	public void mapToGrinTaxonomy() throws Exception {
		LOG.info("Mapping MCPD Taxonomy2 to GRIN Taxonomy");
		genesysTaxonomy2GRIN.update();
		taxonomyService.updateFamilyNames();
		LOG.info("Updating done");
	}

	@PostMapping(value = "/pdci/institute-pdci")
	public void updatePDCI() {
		for (FaoInstitute institute : instituteService.listActive(PageRequest.of(0, Integer.MAX_VALUE))) {
			LOG.info("Updating PDCI for {}", institute.getCode());
			accessionCounter.recountInstitute(institute);
		}
	}

	@PostMapping(value = "/pdci/update")
	public void updateFilteredPDCI(@RequestBody AccessionFilter filter) throws Exception {

		LOG.warn("Recalculating PDCI for accessions matching filter: {}", filter);

		accessionProcessor.apply(filter, (accessions) -> {

			// Everything here is executed within a @Transaction(readOnly = false) context
			if (accessions == null) {
				return;
			}

			accessions.forEach(accession -> {
				AccessionId accessionId = accession.getAccessionId();
				PDCI pdci = accessionId.getPdci();
				// create new PDCI if missing
				PDCI resultingPdci = PDCICalculator.updatePdci(pdci == null ? new PDCI() : pdci, accession);

				updateAccessionPDCI(accession, resultingPdci);
				accessionCounter.recountInstitute(accession.getInstitute());
			});

			LOG.debug("Updated {} PDCI entries", accessions.size());
		});

	}

	private void updateAccessionPDCI(Accession accession, PDCI pdci) {
		AccessionId accessionId = accession.getAccessionId();

		pdci.setAccession(accessionId);
		pdciRepository.save(pdci);

		if (accessionId.getPdci() == null) {
			accessionId.setPdci(pdci);
			LOG.trace("Assigning new PDCI for {}", accession);
			jpaQueryFactory.update(QAccessionId.accessionId).where(QAccessionId.accessionId.eq(accessionId)).set(QAccessionId.accessionId.pdci(), pdci).execute();

			// jpaQueryFactory.update(QAccession.accession).where(QAccession.accession.eq(accession)).set(QAccession.accession.pdci,
			// pdci).execute();
		}
	}

	@PostMapping(value = "/update-tile-index")
	public void updateTileIndex(@RequestBody AccessionFilter filter) throws Exception {

		LOG.warn("Recalculating tileIndex for accessions matching filter: {}", filter);

		AtomicLong updates = new AtomicLong();
		accessionProcessor.apply(filter, (accessions) -> {
			if (accessions == null) {
				return;
			}

			ArrayList<AccessionId> toSave = new ArrayList<>();
			accessions.forEach(accession -> {
				AccessionId accessionId = accession.getAccessionId();
				var ti = WorldClimUtil.getWorldclim25Tile(accessionId.getLongitude(), accessionId.getLatitude());
				var ti3 = TileIndexCalculator.get3MinuteTileIndex(accessionId.getLongitude(), accessionId.getLatitude());

				if (! Objects.equals(ti, accessionId.getTileIndex()) || ! Objects.equals(ti3, accessionId.getTileIndex3min())) {
					if (LOG.isTraceEnabled()) LOG.trace("{}\t\t{}!={}\t\t\t{}!={}", accessionId.getId(), ti, accessionId.getTileIndex(), ti3, accessionId.getTileIndex3min());
					accessionId.setTileIndex(ti);
					accessionId.setTileIndex3min(ti3);
					toSave.add(accessionId);
				}
			});

			accessionIdRepository.saveAll(toSave);
			updates.addAndGet(toSave.size());
			if (toSave.size() > 0 && LOG.isDebugEnabled()) {
				LOG.debug("Assigned {}/{} tileIndexes", toSave.size(), accessions.size());
			}
		});
	}

	@PostMapping(value = "/geo/update-georegions")
	public void updateGeoReg() throws Exception {
		geoRegionService.updateGeoRegionData();
	}

	@PostMapping(value = "/taxonomy/taxonomy-cleanup")
	public void cleanupTaxonomies() {
		taxonomyService.cleanupTaxonomies();
	}

	@PostMapping(value = "/cropname-crop")
	public void assignCropWithCropname() {
		LOG.info("Assigning crops to accessions with CROPNAME.");
		AtomicLong counter = new AtomicLong(0);
		StopWatch stopWatch = new StopWatch();
		stopWatch.start();
		final Set<Long> batch = Collections.synchronizedSet(new HashSet<>(100));
		List<Long> list = accessionRepository.listAccessionIdsWithCropname();
		LOG.info("The list has {} elements", list.size());
		list.stream().parallel().forEach(accessionId -> {
			batch.add(accessionId);
			Set<Long> copy = null;
			synchronized (batch) {
				if (batch.size() > 100) {
					copy = new HashSet<>(batch);
					batch.clear();
				}
			}
			if (copy != null)
				genesysService.updateAccessionCrops(copy);
			if (counter.incrementAndGet() % 1000 == 0 && LOG.isInfoEnabled()) {
				LOG.info("Updated {} records overall_rate={} records/s", counter.get(), Math.round(1000.0 * counter.get() / (stopWatch.getTime())));
			}
		});

		// handle remaining
		genesysService.updateAccessionCrops(batch);

		LOG.info("Done assigning crops to accessions with CROPNAME.");
	}

	@PostMapping(value = "/clear-dois")
	public void clearDois() {
		LOG.info("Clear DOIs");
		genesysService.removeDOIs();
	}

	@PostMapping(value = "/reindex-es")
	public void reindexElasticsearch() {
		LOG.info("Reindex Elasticsearch");
		elasticsearchService.reindexAll();
	}

	@PostMapping(value = "/fix-repo/accession-folders")
	@Transactional
	public void updateAccessionFolders() throws Exception {
		LOG.warn("Registering RepositoryFolders with Accessions");

		repositoryService.getFolder(Paths.get("/wiews")).getChildren().forEach(wiewsFolder -> {
			try {
				String instCode = wiewsFolder.getName();
				RepositoryFolder instAccnFolders = repositoryService.getFolder(wiewsFolder.getFolderPath().resolve("acn"));
				if (instAccnFolders != null) {
					LOG.warn("Processing acn folders for {}", instCode);
					instAccnFolders.getChildren().forEach(acceFolder -> {
						try {
							Accession accession = genesysService.getAccession(instCode, acceFolder.getName());
							if (accession != null) {
								LOG.debug("Folder for accession {}:{} is {}", instCode, accession.getAccessionNumber(), acceFolder.getPath());
								accession.getAccessionId().setRepositoryFolder(acceFolder);
								{
									// Get number of images in the gallery
									ImageGallery gallery = acceFolder.getGallery();
									if (gallery != null) {
										accession.getAccessionId().setImageCount(gallery.getImages().size());
									}
								}
								accessionIdRepository.save(accession.getAccessionId());
							} else {
								LOG.warn("No accession {}:{} for folder {}", instCode, acceFolder.getName(), acceFolder.getPath());
							}
						} catch (NonUniqueAccessionException e) {
							LOG.warn("Accession not unique {}:{}", instCode, acceFolder.getName());
						}
					});
				} else {
					LOG.info("No /wiews/{}/acn folder", instCode);
				}
			} catch (InvalidRepositoryPathException e) {
				LOG.warn("Invalid path {}", e.getMessage());
			}
		});
	}

	@PostMapping(value = "/updateClimate")
	public void worldClim() throws IOException {
		worldClimUpdater.update();
	}


	@PostMapping(value = "/institute/change-instcode")
	@Transactional
	public void changeInstituteCode(@RequestBody FaoInstitute institute) {
		FaoInstitute currentInstitute = instituteService.get(institute.getId());
		adminService.changeInstituteCode(currentInstitute.getCode(), institute.getCode());
	}

	@PostMapping(value = "/kill")
	@Transactional
	public void kill() {
		LOG.error("Killing the server");
		System.exit(-666);
	}

	@Autowired
	private RepositoryFolderRepository folderRepository;
	@Autowired
	private RepositoryFilePersistence fileRepository;
	@Autowired
	private ImageGalleryPersistence imageGalleryRepository;
	@Autowired
	private SubsetRepository subsetRepository;

	@PostMapping(value = "/institute/fix-acl")
	@Transactional
	public void aclFixInstitutesAcl() throws Exception {
		LOG.warn("Adding ACL for FaoInstitutes");

		instituteRepository.findAll().forEach(institute -> {
			// LOG.warn("Making FaoInstitute {} public", institute.getCode());
			aclService.createOrUpdatePermissions(institute);
		});

		LOG.warn("Added ACL to existing FaoInstitutes");
	}

	@PostMapping(value = "/repository/fix-acl")
	@Transactional
	public void aclFixRepositoryAcl() throws Exception {

		LOG.warn("Adding ACL for Repository folders");
		folderRepository.findAll().forEach(folder -> {
			aclService.createOrUpdatePermissions(folder);
		});

		LOG.warn("Adding ACL for Repository files");
		fileRepository.findAll().forEach(file -> {
			aclService.createOrUpdatePermissions(file);
		});

		LOG.warn("Adding ACL for Image galleries");
		imageGalleryRepository.findAll().forEach(gallery -> {
			aclService.createOrUpdatePermissions(gallery);
		});
	}

	@PostMapping(value = "/subsets/fix-acl")
	@Transactional
	public void aclFixSubsetAcl() throws Exception {
		LOG.warn("Adding ACL for Subsets");
		subsetRepository.findAll().forEach(subset -> {
			LOG.warn("Setting ACL for Subset {}", subset.getTitle());
			aclService.makePubliclyReadable(subset, subset.isPublished());
		});
	}

	@PostMapping(value = "/dataset/fix-acl")
	@Transactional
	public void aclFixDatasetAcl() throws Exception {

		LOG.warn("Adding ACL for Datasets");
		datasetRepository.findAll().forEach(dataset -> {
			LOG.warn("Setting ACL for Dataset {}", dataset.getTitle());
			aclService.makePubliclyReadable(dataset, dataset.isPublished());
		});
	}

	@PostMapping(value = "/kpi/fix-acl")
	@Transactional
	public void aclFixKPIAcl() throws Exception {
		LOG.warn("Adding ACL support to KPI Execution");

		kpiExecutionRepository.findAll().forEach(execution -> {
			LOG.warn("Making KPI Execution {} ACL-ready", execution.getName());
			aclService.createOrUpdatePermissions(execution);
		});
	}

	@PostMapping(value = "/threaddump", produces = { MediaType.TEXT_PLAIN_VALUE })
	public String threadDump() throws IOException {

		LOG.warn("Dumping thread info");
		StringBuilder writer = new StringBuilder();
		writer.append("Thread dump:\t").append(Instant.now()).append(System.lineSeparator());
		writer.append("Hostname:\t").append(java.net.InetAddress.getLocalHost().getHostName()).append(System.lineSeparator());

		writer.append("CPUs:\t").append(Runtime.getRuntime().availableProcessors()).append(System.lineSeparator());
		writer.append("Free memory:\t").append(Runtime.getRuntime().freeMemory()).append(System.lineSeparator());
		writer.append("Max memory:\t").append(Runtime.getRuntime().maxMemory()).append(System.lineSeparator());
		writer.append("Total memory:\t").append(Runtime.getRuntime().totalMemory()).append(System.lineSeparator());
		writer.append(System.lineSeparator());

		Map<Thread, StackTraceElement[]> threadSet = Thread.getAllStackTraces();

		ArrayList<Thread> sortedThreads = new ArrayList<>(threadSet.keySet());
		sortedThreads.sort((t1, t2) -> t1.getName().compareTo(t2.getName()));

		writer.append("\n\n*** Thread list ***\n");

		for (Thread t : sortedThreads) {
			writer.append(t.getName());
		}
		writer.append(System.lineSeparator()).append(System.lineSeparator());

		writer.append("\n\n*** Threads ***\n").append(System.lineSeparator());
		writer.append("ID\tState\tName\tGroup").append(System.lineSeparator());
		for (Thread t : sortedThreads) {
			writer.append(t.getId());
			writer.append("\t");
			writer.append(t.getState());
			writer.append("\t");
			writer.append(t.getName());
			writer.append("\t");
			ThreadGroup threadGroup = t.getThreadGroup();
			writer.append(threadGroup == null ? "N/A" : threadGroup.getName());
			writer.append(System.lineSeparator());

			StackTraceElement[] ste = threadSet.get(t);
			Arrays.stream(ste).forEach((st) -> {
				writer.append(t.getId());
				writer.append("\t");
				writer.append(st.getClassName());
				writer.append(":");
				writer.append(st.getMethodName());
				writer.append("\t");
				writer.append(StringUtils.defaultIfBlank(st.getFileName(), "---"));
				writer.append("\t");
				writer.append(System.lineSeparator());
			});

			writer.append(System.lineSeparator());
		}

		ThreadMXBean tmxb = ManagementFactory.getThreadMXBean();
		if (tmxb != null && tmxb.isThreadCpuTimeSupported()) {
			writer.append("\n\n*** CPU Usage ***\n").append(System.lineSeparator());
			writer.append("ID\tCPU%\tCPU[ms]\tUser[ms]\tState\tName\tGroup").append(System.lineSeparator());
			Map<Long, ThreadCpuUsage> cpuUsage = findCpuUsage(tmxb, sortedThreads);

			sortedThreads.sort((t1, t2) -> (int)(cpuUsage.get(t2.getId()).cpuTime - cpuUsage.get(t1.getId()).cpuTime));
			for (Thread t : sortedThreads) {
				writer.append(t.getId());
				writer.append("\t");
				ThreadCpuUsage cpu = cpuUsage.get(t.getId());
				writer.append(String.format("%.4f", cpu.utilization * 100));
				writer.append("\t");
				writer.append(String.format("%.2f", cpu.cpuTime / 1000f));
				writer.append("\t");
				writer.append(String.format("%.2f", cpu.userTime / 1000f));
				writer.append("\t");

				writer.append(t.getState());
				writer.append("\t");
				writer.append(t.getName());
				writer.append("\t");
				ThreadGroup threadGroup = t.getThreadGroup();
				writer.append(threadGroup == null ? "N/A" : threadGroup.getName());

				writer.append(System.lineSeparator());
			}
		}
		return writer.toString();
	}

	@PostMapping(value = "/send-email")
	public void sendEmail(@RequestBody @Validated AdminController.SendEmailRequest sendEmailRequest) {
		userService.sendEmail(sendEmailRequest.uuids, sendEmailRequest.template, sendEmailRequest.subject);
	}

	public static class SendEmailRequest {
		@NotNull
		public Set<UUID> uuids;
		@NotNull
		public String template;
		@NotNull
		public String subject;
	}

	private Map<Long, ThreadCpuUsage> findCpuUsage(ThreadMXBean tmxb, List<? extends Thread> threads) {
		Map<Long, ThreadCpuUsage> cpu = new Hashtable<>();
		for (int i = 0; i < threads.size(); i++) {
			Thread thread = threads.get(i);
			long threadId = thread.getId();
			ThreadCpuUsage cpuUsage = new ThreadCpuUsage();
			cpuUsage.cpuTime1 = tmxb.getThreadCpuTime(threadId);
			cpuUsage.userTime1 = tmxb.getThreadUserTime(threadId);
			cpuUsage.time1 = System.currentTimeMillis();
			cpu.put(threadId, cpuUsage);
		}
		try {
			Thread.sleep(2000);
		} catch (InterruptedException e) {
			LOG.warn("Interrupted from sleep: " + e.getMessage());
		}

		for (int i = 0; i < threads.size(); i++) {
			Thread thread = threads.get(i);
			long threadId = thread.getId();
			ThreadCpuUsage cpuUsage = cpu.get(threadId);
			cpuUsage.cpuTime2 = tmxb.getThreadCpuTime(threadId);
			cpuUsage.userTime2 = tmxb.getThreadUserTime(threadId);
			cpuUsage.time2 = System.currentTimeMillis();

			cpuUsage.cpuTime = cpuUsage.cpuTime2 - cpuUsage.cpuTime1;
			cpuUsage.userTime = cpuUsage.userTime2 - cpuUsage.userTime1;
			cpuUsage.time = cpuUsage.time2 - cpuUsage.time1;

//			if (cpuUsage.cpuTime > 0) {
//				cpuUsage.threadInfo = tmxb.getThreadInfo(threadId, 50);
//			}

			cpuUsage.utilization = (cpuUsage.cpuTime) / ((cpuUsage.time) * 1000000F);
		}

		return cpu;
	}

	private static class ThreadCpuUsage {
		public long userTime1, userTime2, userTime;
		public long cpuTime1, cpuTime2, cpuTime;
		public long time1, time2, time;
		public double utilization;
	}
}