Skip to content

Commit eee5411

Browse files
costas80costas80
committed
Pull request #278: Model cleanup and concurrent validation throttling
Merge in ITB/shacl-validator from development to master * commit '8eed2ef6db0cd0077086c6b66c32818f8a89af3f': Model cleanup and concurrent validation throttling
2 parents 7cbd593 + 8eed2ef commit eee5411

File tree

12 files changed

+516
-275
lines changed

12 files changed

+516
-275
lines changed

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/ApplicationConfig.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,21 @@ public class ApplicationConfig extends eu.europa.ec.itb.validation.commons.confi
5353
private String defaultRdfReportSyntaxDescription;
5454
private String defaultRdfReportQueryDescription;
5555
private String defaultLocaleDescription;
56+
private Integer maximumConcurrentValidations;
57+
58+
/**
59+
* @return The maximum number of concurrent validations to perform.
60+
*/
61+
public Integer getMaximumConcurrentValidations() {
62+
return maximumConcurrentValidations;
63+
}
64+
65+
/***
66+
* @param maximumConcurrentValidations The maximum number of concurrent validations to perform.
67+
*/
68+
public void setMaximumConcurrentValidations(Integer maximumConcurrentValidations) {
69+
this.maximumConcurrentValidations = maximumConcurrentValidations;
70+
}
5671

5772
/**
5873
* @return The default web service input description for adding the SHACL validation report to the TAR report context.

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/ModelManager.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright (C) 2025 European Union
3+
*
4+
* Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent
5+
* versions of the EUPL (the "Licence"); You may not use this work except in compliance with the Licence.
6+
*
7+
* You may obtain a copy of the Licence at:
8+
*
9+
* https://interoperable-europe.ec.europa.eu/collection/eupl/eupl-text-eupl-12
10+
*
11+
* Unless required by applicable law or agreed to in writing, software distributed under the Licence is distributed on an
12+
* "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licence for
13+
* the specific language governing permissions and limitations under the Licence.
14+
*/
15+
16+
package eu.europa.ec.itb.shacl;
17+
18+
import eu.europa.ec.itb.shacl.validation.FileManager;
19+
import org.apache.jena.rdf.model.Model;
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
22+
23+
import java.util.Collections;
24+
import java.util.IdentityHashMap;
25+
import java.util.Set;
26+
27+
/**
28+
* Class used to track Jena RDF Model instances in use and ensure their cleanup.
29+
*/
30+
public class ModelManager {
31+
32+
private static final Logger LOG = LoggerFactory.getLogger(ModelManager.class);
33+
34+
private final FileManager fileManager;
35+
private final Set<Model> trackedModels = Collections.newSetFromMap(new IdentityHashMap<>());
36+
37+
/**
38+
* Constructor.
39+
*
40+
* @param fileManager The file manager to use.
41+
*/
42+
public ModelManager(FileManager fileManager) {
43+
this.fileManager = fileManager;
44+
}
45+
46+
/**
47+
* Track the provided model.
48+
*
49+
* @param model The model to track.
50+
*/
51+
public void track(Model model) {
52+
if (model != null) {
53+
if (fileManager.isCachedModel(model)) {
54+
if (LOG.isDebugEnabled()) {
55+
LOG.debug("Model not tracked as it is cached");
56+
}
57+
} else {
58+
if (LOG.isDebugEnabled()) {
59+
LOG.debug("Tracking model");
60+
}
61+
trackedModels.add(model);
62+
}
63+
}
64+
}
65+
66+
/**
67+
* Close all tracked models.
68+
*/
69+
public void close() {
70+
if (LOG.isDebugEnabled()) {
71+
LOG.debug("Closing {} models", trackedModels.size());
72+
}
73+
try {
74+
trackedModels.stream().filter(model -> model != null && !fileManager.isCachedModel(model) && !model.isClosed()).forEach(Model::close);
75+
} catch (Exception e) {
76+
LOG.warn("Error while closing tracked model", e);
77+
}
78+
}
79+
80+
}

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/ValidationSpecs.java

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import eu.europa.ec.itb.validation.commons.FileInfo;
1919
import eu.europa.ec.itb.validation.commons.LocalisationHelper;
20+
import org.apache.jena.rdf.model.Model;
2021

2122
import java.io.File;
2223
import java.util.List;
@@ -36,6 +37,7 @@ public class ValidationSpecs {
3637
private LocalisationHelper localiser;
3738
private boolean logProgress;
3839
private boolean usePlugins;
40+
private ModelManager modelManager;
3941

4042
/**
4143
* Private constructor to prevent direct initialisation.
@@ -105,6 +107,15 @@ public boolean isUsePlugins() {
105107
return usePlugins;
106108
}
107109

110+
/**
111+
* Track the provided model.
112+
*
113+
* @param model The model to track.
114+
*/
115+
public void track(Model model) {
116+
if (modelManager != null) modelManager.track(model);
117+
}
118+
108119
/**
109120
* Build the validation specifications.
110121
*
@@ -115,13 +126,14 @@ public boolean isUsePlugins() {
115126
* @param loadImports True if OWL imports in the content should be loaded before validation.
116127
* @param domainConfig The domain in question.
117128
* @param localiser Helper class for localisations.
129+
* @param modelManager The model manager instance to use.
118130
* @return The specification builder.
119131
*/
120-
public static Builder builder(File inputFileToValidate, String validationType, String contentSyntax, List<FileInfo> externalShaclFiles, boolean loadImports, DomainConfig domainConfig, LocalisationHelper localiser) {
132+
public static Builder builder(File inputFileToValidate, String validationType, String contentSyntax, List<FileInfo> externalShaclFiles, boolean loadImports, DomainConfig domainConfig, LocalisationHelper localiser, ModelManager modelManager) {
121133
if (validationType == null) {
122134
validationType = domainConfig.getType().get(0);
123135
}
124-
return new Builder(inputFileToValidate, validationType, contentSyntax, externalShaclFiles, loadImports, domainConfig, localiser);
136+
return new Builder(inputFileToValidate, validationType, contentSyntax, externalShaclFiles, loadImports, domainConfig, localiser, modelManager);
125137
}
126138

127139
/**
@@ -135,14 +147,15 @@ public static class Builder {
135147
* Constructor.
136148
*
137149
* @param inputFileToValidate The input RDF (or other) content to validate.
138-
* @param validationType The type of validation to perform.
139-
* @param contentSyntax The mime type of the provided RDF content.
140-
* @param externalShaclFiles Any shapes to consider that are externally provided
141-
* @param loadImports True if OWL imports in the content should be loaded before validation.
142-
* @param domainConfig The domain in question.
143-
* @param localiser Helper class for localisations.
150+
* @param validationType The type of validation to perform.
151+
* @param contentSyntax The mime type of the provided RDF content.
152+
* @param externalShaclFiles Any shapes to consider that are externally provided
153+
* @param loadImports True if OWL imports in the content should be loaded before validation.
154+
* @param domainConfig The domain in question.
155+
* @param localiser Helper class for localisations.
156+
* @param modelManager The model manager instance to use.
144157
*/
145-
Builder(File inputFileToValidate, String validationType, String contentSyntax, List<FileInfo> externalShaclFiles, boolean loadImports, DomainConfig domainConfig, LocalisationHelper localiser) {
158+
Builder(File inputFileToValidate, String validationType, String contentSyntax, List<FileInfo> externalShaclFiles, boolean loadImports, DomainConfig domainConfig, LocalisationHelper localiser, ModelManager modelManager) {
146159
instance = new ValidationSpecs();
147160
this.instance.contentSyntax = contentSyntax;
148161
this.instance.inputFileToValidate = inputFileToValidate;
@@ -153,6 +166,7 @@ public static class Builder {
153166
this.instance.validationType = validationType;
154167
this.instance.logProgress = true;
155168
this.instance.usePlugins = true;
169+
this.instance.modelManager = modelManager;
156170
}
157171

158172
/**

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/config/ResourcePreloader.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,7 @@
1515

1616
package eu.europa.ec.itb.shacl.config;
1717

18-
import eu.europa.ec.itb.shacl.ApplicationConfig;
19-
import eu.europa.ec.itb.shacl.DomainConfig;
20-
import eu.europa.ec.itb.shacl.DomainConfigCache;
21-
import eu.europa.ec.itb.shacl.ValidationSpecs;
18+
import eu.europa.ec.itb.shacl.*;
2219
import eu.europa.ec.itb.shacl.validation.FileManager;
2320
import eu.europa.ec.itb.shacl.validation.SHACLValidator;
2421
import eu.europa.ec.itb.validation.commons.LocalisationHelper;
@@ -81,15 +78,16 @@ private void preloadImportsAndShapeGraphs() {
8178
LOG.info("Preloading owl:import references for domain [{}]", domainConfig.getDomainName());
8279
var localiser = new LocalisationHelper(domainConfig, Utils.getSupportedLocale(null, domainConfig));
8380
// Simulate a validation for each validation type with an empty model.
84-
Model emptyModel = ModelFactory.createDefaultModel();
8581
String contentType = RDFLanguages.TURTLE.getContentType().getContentTypeStr();
8682
// Iterate over validation types.
83+
var modelManager = new ModelManager(fileManager);
8784
domainConfig.getType().stream().filter(domainConfig::preloadImportsForType).forEach(validationType -> {
8885
Path parentFolder = Path.of(appConfig.getTmpFolder(), UUID.randomUUID().toString());
8986
boolean mergeModelsBeforeValidationSetting = domainConfig.isMergeModelsBeforeValidation();
9087
try {
9188
Files.createDirectories(parentFolder);
9289
Path inputFile = Files.createFile(parentFolder.resolve("emptyModel.ttl"));
90+
Model emptyModel = ModelFactory.createDefaultModel();
9391
try (var out = Files.newOutputStream(inputFile)) {
9492
emptyModel.write(out, RDFLanguages.TURTLE.getName());
9593
out.flush();
@@ -98,7 +96,7 @@ private void preloadImportsAndShapeGraphs() {
9896
domainConfig.setMergeModelsBeforeValidation(false);
9997
try {
10098
LOG.info("Preloading owl:import references for validation type [{}]", validationType);
101-
ValidationSpecs specs = ValidationSpecs.builder(inputFile.toFile(), validationType, contentType, Collections.emptyList(), false, domainConfig, localiser)
99+
ValidationSpecs specs = ValidationSpecs.builder(inputFile.toFile(), validationType, contentType, Collections.emptyList(), false, domainConfig, localiser, modelManager)
102100
.withoutPlugins()
103101
.withoutProgressLogging()
104102
.build();
@@ -115,6 +113,7 @@ private void preloadImportsAndShapeGraphs() {
115113
} catch (Exception e) {
116114
LOG.warn("Failed to preload OWL imports for domain [{}]", domainConfig.getDomainName(), e);
117115
} finally {
116+
modelManager.close();
118117
FileUtils.deleteQuietly(parentFolder.toFile());
119118
// Restore merge model setting.
120119
domainConfig.setMergeModelsBeforeValidation(mergeModelsBeforeValidationSetting);

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/validation/FileManager.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,7 @@
5454
import java.net.URI;
5555
import java.nio.file.Files;
5656
import java.nio.file.Path;
57-
import java.util.List;
58-
import java.util.Map;
59-
import java.util.Objects;
60-
import java.util.UUID;
57+
import java.util.*;
6158
import java.util.concurrent.ConcurrentHashMap;
6259
import java.util.concurrent.locks.ReentrantLock;
6360
import java.util.function.Supplier;
@@ -79,6 +76,7 @@ public class FileManager extends BaseFileManager<ApplicationConfig> {
7976

8077
private final ConcurrentHashMap<String, Path> shaclModelCache = new ConcurrentHashMap<>();
8178
private final ConcurrentHashMap<String, Model> materialisedShaclModelCache = new ConcurrentHashMap<>();
79+
private final Set<Model> cachedModels = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
8280
private final ReentrantLock cacheLock = new ReentrantLock();
8381

8482
/**
@@ -212,6 +210,7 @@ public void writeShaclShapes(Path outputPath, Model rdfModel, String validationT
212210
}
213211
shaclModelCache.put(cacheKey, cachedPath);
214212
materialisedShaclModelCache.put(cacheKey, rdfModel);
213+
cachedModels.add(rdfModel);
215214
}
216215
} finally {
217216
cacheLock.unlock();
@@ -259,6 +258,16 @@ public Model readModel(InputStream dataStream, Lang rdfLanguage, Map<String, Str
259258
return builder.build().toModel();
260259
}
261260

261+
/**
262+
* Check to see if the provided model is cached.
263+
*
264+
* @param modelToCheck The model to check.
265+
* @return The check result.
266+
*/
267+
public boolean isCachedModel(Model modelToCheck) {
268+
return modelToCheck != null && cachedModels.contains(modelToCheck);
269+
}
270+
262271
/**
263272
* Add the provided model to the cache if possible.
264273
*
@@ -271,6 +280,7 @@ private void cacheShapeModelIfPossible(ValidationSpecs specs, Model shapeModel)
271280
cacheLock.lock();
272281
try {
273282
materialisedShaclModelCache.putIfAbsent(cacheKey, shapeModel);
283+
cachedModels.add(shapeModel);
274284
} finally {
275285
cacheLock.unlock();
276286
}

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/validation/SHACLValidator.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public ModelPair validateAll() {
121121
if (specs.isLogProgress()) {
122122
LOG.info("Starting validation..");
123123
}
124-
try {
124+
try {
125125
Model validationReport = validateAgainstShacl();
126126
if (specs.isUsePlugins()) {
127127
validateAgainstPlugins(validationReport);
@@ -374,6 +374,9 @@ private Model validateShacl(List<FileInfo> shaclFiles) {
374374
Resource resource = ValidationUtil.validateModel(dataModel, this.aggregatedShapes, false);
375375
reportModel = resource.getModel();
376376
}
377+
specs.track(this.dataModel);
378+
specs.track(this.aggregatedShapes);
379+
specs.track(reportModel);
377380
reportModel.setNsPrefix("sh", SHACLResources.NS_SHACL);
378381
return reportModel;
379382
}
@@ -419,6 +422,7 @@ private Model getShapesModel(List<FileInfo> shaclFiles) {
419422
}
420423

421424
this.importedShapes = JenaUtil.createMemoryModel();
425+
specs.track(this.importedShapes);
422426
createImportedModels(aggregateModel);
423427
if (this.importedShapes != null) {
424428
aggregateModel.add(importedShapes);

‎shaclvalidator-common/src/main/java/eu/europa/ec/itb/shacl/validation/ThroughputThrottler.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright (C) 2025 European Union
3+
*
4+
* Licensed under the EUPL, Version 1.2 or - as soon they will be approved by the European Commission - subsequent
5+
* versions of the EUPL (the "Licence"); You may not use this work except in compliance with the Licence.
6+
*
7+
* You may obtain a copy of the Licence at:
8+
*
9+
* https://interoperable-europe.ec.europa.eu/collection/eupl/eupl-text-eupl-12
10+
*
11+
* Unless required by applicable law or agreed to in writing, software distributed under the Licence is distributed on an
12+
* "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Licence for
13+
* the specific language governing permissions and limitations under the Licence.
14+
*/
15+
16+
package eu.europa.ec.itb.shacl.validation;
17+
18+
import eu.europa.ec.itb.shacl.ApplicationConfig;
19+
import jakarta.annotation.PostConstruct;
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
22+
import org.springframework.beans.factory.annotation.Autowired;
23+
import org.springframework.stereotype.Component;
24+
25+
import java.util.concurrent.Semaphore;
26+
import java.util.function.Supplier;
27+
28+
/**
29+
* Singleton component used to apply a throughput rate on concurrently executing validations.
30+
* <p/>
31+
* The implementation follows a simple locking approach given that validations cannot be executed asynchronously.
32+
*/
33+
@Component
34+
public class ThroughputThrottler {
35+
36+
private static final Logger LOG = LoggerFactory.getLogger(ThroughputThrottler.class);
37+
38+
@Autowired
39+
private ApplicationConfig appConfig;
40+
private Semaphore semaphore;
41+
42+
@PostConstruct
43+
public void initialise() {
44+
Integer configuredMaximum = appConfig.getMaximumConcurrentValidations();
45+
if (configuredMaximum != null && configuredMaximum > 0) {
46+
// Create a FIFO semaphore with the configured capacity.
47+
semaphore = new Semaphore(configuredMaximum, true);
48+
LOG.info("Validation throttling allows {} validations to proceed in parallel", configuredMaximum);
49+
} else {
50+
semaphore = null;
51+
LOG.info("No validation throttling configured");
52+
}
53+
}
54+
55+
/**
56+
* Proceed with the supplied task while respecting configured throttling.
57+
*
58+
* @param task The validation task to execute.
59+
* @return The result.
60+
* @param <T> The type of result.
61+
*/
62+
public <T> T proceed(Supplier<T> task) {
63+
if (semaphore == null) {
64+
// No throttling.
65+
return task.get();
66+
} else {
67+
try {
68+
semaphore.acquire();
69+
if (LOG.isDebugEnabled()) {
70+
LOG.debug("Thread [{}] acquired semaphore permit ({} permits left - current queue {})", Thread.currentThread().getName(), semaphore.availablePermits(), semaphore.getQueueLength());
71+
}
72+
return task.get();
73+
} catch (InterruptedException e) {
74+
Thread.currentThread().interrupt();
75+
throw new IllegalStateException("Interrupted thread while waiting for throughput throttling", e);
76+
} finally {
77+
if (LOG.isDebugEnabled()) {
78+
LOG.debug("Thread [{}] releasing semaphore lock", Thread.currentThread().getName());
79+
}
80+
semaphore.release();
81+
}
82+
}
83+
}
84+
85+
}

0 commit comments

Comments
 (0)