DataReader.java
/*
* Copyright 2013 University of Glasgow.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package broadwick.data;
import broadwick.data.readers.FullMovementsFileReader;
import broadwick.data.readers.PopulationsFileReader;
import broadwick.data.readers.BatchedMovementsFileReader;
import broadwick.data.readers.LocationsFileReader;
import broadwick.data.readers.DirectedMovementsFileReader;
import broadwick.data.readers.TestsFileReader;
import broadwick.BroadwickException;
import broadwick.config.generated.DataFiles;
import broadwick.config.generated.DataFiles.BatchMovementFile;
import broadwick.config.generated.DataFiles.DirectedMovementFile;
import broadwick.config.generated.DataFiles.FullMovementFile;
import broadwick.config.generated.DataFiles.LocationsFile;
import broadwick.config.generated.DataFiles.PopulationFile;
import broadwick.config.generated.DataFiles.TestsFile;
import broadwick.config.generated.Project;
import java.util.List;
import java.util.Map;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.time.StopWatch;
/**
* Utility class to read, store and manipulate the data files.
*/
@Slf4j
public class DataReader implements java.lang.AutoCloseable {
/**
* Create the object that will read the data element of the configuration file.
* @param data the <data/> element of the configuration file.
*/
public DataReader(final Project.Data data) {
this.data = data;
// if there is a data section in the config file let's read it.
if (data != null) {
if (data.getDatabases() != null) {
// there is a database mentioned in the config file so let's use it
dbName = data.getDatabases().getName();
// dbImpl = new FirebirdDatabase(dbName, false);
// dbImpl = new DerbyDatabase(dbName, false);
// dbImpl = new HyperSqlDatabase(dbName, false);
dbImpl = new H2Database(dbName, false);
} else if (data.getDatafiles() != null) {
// there is a datafiles section in the config file so we will read them and create a randomly named
// database.
final String db = "broadwick_" + RandomStringUtils.randomAlphanumeric(8);
dbName = db + "_db";
// dbImpl = new FirebirdDatabase(dbName, true);
// dbImpl = new DerbyDatabase(dbName, true);
// dbImpl = new HyperSqlDatabase(dbName, true);
dbImpl = new H2Database(dbName, true);
readDataSection();
} else {
throw new BroadwickException("There is a <data> section in the configuration file but neither a <database> nor <datafiles> section.");
}
lookup = new Lookup(dbImpl);
logDbStatistics();
}
}
/**
* Read the data section from the configuration file. If a database section is found then it is used and any files
* specified are ignored.
*/
public final void readDataSection() {
final StopWatch sw = new StopWatch();
final DataFiles files = data.getDatafiles();
if (files != null) {
sw.start();
readDataFiles(files);
sw.stop();
log.info("Processed input data in {} ms", sw.toString());
log.info("Data stored internally in {}.", dbName);
}
}
@Override
public final void close() {
if (dbImpl != null) {
log.trace("Closing database connection");
dbImpl.close();
}
}
/**
* Save a message giving the numbers of elements in the database.
*/
private void logDbStatistics() {
log.info("Read {} population data from the database.", lookup.getNumAnimals());
log.info("Read {} locations data from the database.", lookup.getNumLocations());
log.info("Read {} tests data from the database.", lookup.getNumTests());
log.info("Read {} movements data from the database.", lookup.getNumMovements());
}
/**
* Read the datafiles, each section type of the <datafiles/> element (e.g. all the <MovementFile/>s) are read in
* parallel and each individual element is its read in parallel with the data saved to an internal database. Once
* the datafiles have been read some post-processing is run to determine the locations of individuals at each time
* step throughout the simulation.
* @param files the Datafiles object from the configuration file.
*/
private void readDataFiles(final DataFiles files) {
try {
final String addingFileMsg = "Reading %s ...";
readAllLocationSections(files.getLocationsFile(), addingFileMsg);
readAllPopulationSections(files.getPopulationFile(), addingFileMsg);
readAllTestSections(files.getTestsFile(), addingFileMsg);
readAllFullMovementSections(files.getFullMovementFile(), addingFileMsg);
readAllDirectedMovementSections(files.getDirectedMovementFile(), addingFileMsg);
readAllBatchedMovementSections(files.getBatchMovementFile(), addingFileMsg);
} catch (Exception e) {
log.error("Failure reading data file section. {}", e.getLocalizedMessage());
throw new BroadwickException(String.format("Failure reading data file section. %s", e.getLocalizedMessage()));
}
}
/**
* Add a description of a configuration file section to an internal key value pair so that the correct column of a
* data file can be read. Each <datafile> section in the configuration contains an element and the column location
* in the data file where this element can be found. This method update a key-value pair of the element name and
* location, recording errors as it finds them.
* @param elementName the name of the element in the configuration file.
* @param indexInDataFile the location of the element in each row of the data file.
* @param keyValueMapping a table of the keyValueMapping name index and type.
* @param errors a description of any parsing errors found.
* @param recordErrors flag to control whether or not error are recorded in the errors parameter.
* @param sectionName the name of the section from which the element being added to the table has been read.
*/
protected final void updateSectionDefiniton(final String elementName, final int indexInDataFile,
final Map<String, Integer> keyValueMapping, final StringBuilder errors,
final boolean recordErrors, final String sectionName) {
if (indexInDataFile != 0) {
keyValueMapping.put(elementName, indexInDataFile);
} else {
if (recordErrors) {
errors.append("No ").append(elementName).append(" column set in ").append(sectionName).append(" section\n");
}
}
}
/**
* Read the directed movements sections of the configuration file.
* @param directedMovementFiles a collection of directedMovement sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of movements read.
*/
private int readAllDirectedMovementSections(final List<DirectedMovementFile> directedMovementFiles, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.DirectedMovementFile file : directedMovementFiles) {
log.trace(String.format(addingFileMsg, file.getName()));
final DirectedMovementsFileReader movementsFileReader = new DirectedMovementsFileReader(file, dbImpl);
elementsRead += movementsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} directed movements from {} files.", elementsRead, directedMovementFiles.size());
}
return elementsRead;
}
/**
* Read the full movements sections of the configuration file.
* @param fullMovementFiles a collection of fullMovement sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of movements read.
*/
private int readAllFullMovementSections(final List<FullMovementFile> fullMovementFiles, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.FullMovementFile file : fullMovementFiles) {
log.trace(String.format(addingFileMsg, file.getName()));
final FullMovementsFileReader movementsFileReader = new FullMovementsFileReader(file, dbImpl);
elementsRead += movementsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} full movements from {} files.", elementsRead, fullMovementFiles.size());
}
return elementsRead;
}
/**
* Read the batched movements sections of the configuration file.
* @param batchMovementFiles a collection of batchMovement sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of movements read.
*/
private int readAllBatchedMovementSections(final List<BatchMovementFile> batchMovementFiles, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.BatchMovementFile file : batchMovementFiles) {
log.trace(String.format(addingFileMsg, file.getName()));
final BatchedMovementsFileReader movementsFileReader = new BatchedMovementsFileReader(file, dbImpl);
elementsRead += movementsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} batched ovements from {} files.", elementsRead, batchMovementFiles.size());
}
return elementsRead;
}
/**
* Read the location sections of the configuration file.
* @param locationsFile a collection of locations sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of locations read.
*/
private int readAllLocationSections(final List<LocationsFile> locationsFile, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.LocationsFile file : locationsFile) {
log.trace(String.format(addingFileMsg, file.getName()));
final LocationsFileReader locationsFileReader = new LocationsFileReader(file, dbImpl);
elementsRead += locationsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} locations from {} files.", elementsRead, locationsFile.size());
}
return elementsRead;
}
/**
* Read the test sections of the configuration file.
* @param testsFile a collection of locations sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of tests read.
*/
private int readAllTestSections(final List<TestsFile> testsFile, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.TestsFile file : testsFile) {
log.trace(String.format(addingFileMsg, file.getName()));
final TestsFileReader testsFileReader = new TestsFileReader(file, dbImpl);
elementsRead += testsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} tests from {} files.", elementsRead, testsFile.size());
}
return elementsRead;
}
/**
* Read the populations sections of the configuration file.
* @param populationsFiles a collection of populations sections.
* @param addingFileMsg formatted string (in the form "Adding %s to reading list") to be added to log files.
* @return the number of populations read.
*/
private int readAllPopulationSections(final List<PopulationFile> populationsFiles, final String addingFileMsg) {
int elementsRead = 0;
for (DataFiles.PopulationFile file : populationsFiles) {
log.trace(String.format(addingFileMsg, file.getName()));
final PopulationsFileReader populationsFileReader = new PopulationsFileReader(file, dbImpl);
elementsRead += populationsFileReader.insert();
}
if (elementsRead > 0) {
log.info("Read {} population data from {} files.", elementsRead, populationsFiles.size());
}
return elementsRead;
}
@SuppressWarnings("PMD.UnusedPrivateField")
@Getter
private Lookup lookup;
private Project.Data data;
private DatabaseImpl dbImpl;
private String dbName;
}