Skip to content

Commit

Permalink
ODRE import postprocessor fixes (powsybl#3080)
Browse files Browse the repository at this point in the history 
* Record LineGeoData
* Reading substations once instead of twice
* Do one graph traversal instead of three
* Use SimpleGraph instead of PseudoGraph
* Not calling twice containsVertex when creating graph
* Avoid duplication of same coordinates
* Only create graph if needed
* Fix substations not filled
* Not opening twice the file
* Avoid list duplication
* Remove LineGeoData/SubstationGeoData classes
* Not reversing coordinates in case of equal distances

Signed-off-by: Florian Dupuy <[email protected]>
  • Loading branch information
@flo-dup
flo-dup authored Sep 23, 2024
1 parent 1e1884a commit f9a167d
Show file tree
Hide file tree
Showing 21 changed files with 387 additions and 573 deletions.
59 changes: 0 additions & 59 deletions iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java
View file

This file was deleted.

38 changes: 0 additions & 38 deletions iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/SubstationGeoData.java
View file

This file was deleted.

141 changes: 24 additions & 117 deletions iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/FileValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,13 @@
*/
package com.powsybl.iidm.geodata.odre;

import com.powsybl.iidm.geodata.utils.InputUtils;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.List;
import java.util.Map;

/**
* @author Ahmed Bendaamer {@literal <ahmed.bendaamer at rte-france.com>}
Expand All @@ -32,133 +24,48 @@ public final class FileValidator {
private FileValidator() {
}

private static final String HEADERS_OF_FILE_HAS_CHANGED = "Invalid file, Headers of file {} has changed, header(s) not found: {}";
private static final String HEADERS_OF_FILE_HAS_CHANGED = "Invalid {} file, header(s) not found: {}";
private static final Logger LOGGER = LoggerFactory.getLogger(FileValidator.class);
static final String COUNTRY_FR = "FR";

private static CSVFormat.Builder createCsvFormatBuilder() {
return CSVFormat.DEFAULT.builder()
.setQuote('"')
.setDelimiter(";")
.setRecordSeparator(System.lineSeparator());
}

static final CSVFormat CSV_FORMAT = createCsvFormatBuilder()
static final CSVFormat CSV_FORMAT = CSVFormat.DEFAULT.builder()
.setQuote('"')
.setDelimiter(";")
.setRecordSeparator(System.lineSeparator())
.setHeader()
.setSkipHeaderRecord(true)
.build();
static final CSVFormat CSV_FORMAT_FOR_HEADER = createCsvFormatBuilder().build();

public static final String SUBSTATIONS = "substations";
public static final String AERIAL_LINES = "aerial-lines";
public static final String UNDERGROUND_LINES = "underground-lines";

public static boolean validateSubstations(Path path, OdreConfig odreConfig) {
try (Reader reader = new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))) {
Iterator<CSVRecord> records = CSVParser.parse(reader, FileValidator.CSV_FORMAT_FOR_HEADER).iterator();

List<String> headers;
if (records.hasNext()) {
CSVRecord headersRecord = records.next();
headers = headersRecord.toList();
} else {
LOGGER.error("The file {} is empty", path.getFileName());
return false;
}

if (new HashSet<>(headers).containsAll(odreConfig.substationsExpectedHeaders())) {
return true;
} else {
List<String> notFoundHeaders = odreConfig.substationsExpectedHeaders().stream().filter(isChangedHeaders(headers)).collect(Collectors.toList());
logHeaderError(path, notFoundHeaders);
}
} catch (IOException e) {
LOGGER.error(e.getMessage());
return false;
}
return false;
public static boolean validateSubstationsHeaders(CSVParser records, OdreConfig odreConfig) {
return validateHeaders(records, odreConfig.substationsExpectedHeaders(), FileValidator.SUBSTATIONS);
}

public static Map<String, Reader> validateLines(List<Path> paths, OdreConfig odreConfig) {
Map<String, Reader> mapResult = new HashMap<>();
paths.forEach(path -> {
try (Reader reader = new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))) {
Iterator<CSVRecord> records = CSVParser.parse(reader, FileValidator.CSV_FORMAT_FOR_HEADER).iterator();

final List<String> headers;
if (records.hasNext()) {
CSVRecord headersRecord = records.next();
headers = headersRecord.toList();
} else {
headers = null;
LOGGER.error("The file {} is empty", path.getFileName());
}

String equipmentType = null;
if (headers != null && records.hasNext()) {
CSVRecord firstRow = records.next();
equipmentType = readEquipmentType(firstRow, headers, odreConfig);
} else {
LOGGER.error("The file {} has no data", path.getFileName());
}

String type = equipmentType != null ? equipmentType : odreConfig.nullEquipmentType();
if (type.equals(odreConfig.nullEquipmentType())) {
getIfSubstationsOrLogError(mapResult, path, headers, equipmentType, odreConfig);
} else if (type.equals(odreConfig.aerialEquipmentType())) {
getResultOrLogError(headers, odreConfig.aerialLinesExpectedHeaders(), mapResult, AERIAL_LINES, path);
} else if (type.equals(odreConfig.undergroundEquipmentType())) {
getResultOrLogError(headers, odreConfig.undergroundLinesExpectedHeaders(), mapResult, UNDERGROUND_LINES, path);
} else {
LOGGER.error("The file {} has no known equipment type : {}", path.getFileName(), equipmentType);
}
} catch (IOException e) {
mapResult.values().forEach(IOUtils::closeQuietly);
throw new UncheckedIOException(e);
}
});
return mapResult;
public static boolean validateAerialLinesHeaders(CSVParser records, OdreConfig odreConfig) {
return validateHeaders(records, odreConfig.aerialLinesExpectedHeaders(), FileValidator.AERIAL_LINES);
}

private static String readEquipmentType(CSVRecord firstRow, List<String> headers, OdreConfig odreConfig) {
String equipmentType = null;
int index = headers.indexOf(odreConfig.equipmentTypeColumn());
if (index != -1) {
equipmentType = firstRow.get(index);
}
return equipmentType;
public static boolean validateUndergroundHeaders(CSVParser records, OdreConfig odreConfig) {
return validateHeaders(records, odreConfig.undergroundLinesExpectedHeaders(), FileValidator.UNDERGROUND_LINES);
}

private static void getIfSubstationsOrLogError(Map<String, Reader> mapResult, Path path, List<String> headers, String typeOuvrage, OdreConfig odreConfig) throws IOException {
if (new HashSet<>(headers).containsAll(odreConfig.substationsExpectedHeaders())) {
mapResult.putIfAbsent(SUBSTATIONS, new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8)));
} else if (isAerialOrUnderground(headers, odreConfig)) {
LOGGER.error("The file {} has no equipment type : {}", path.getFileName(), typeOuvrage);
} else {
List<String> notFoundHeaders = odreConfig.substationsExpectedHeaders().stream().filter(isChangedHeaders(headers)).collect(Collectors.toList());
logHeaderError(path, notFoundHeaders);
private static boolean validateHeaders(CSVParser csvParser, List<String> expectedHeaders, String fileType) {
Map<String, Integer> headerMap = csvParser.getHeaderMap();
if (headerMap.isEmpty()) {
LOGGER.error("The substations file is empty");
return false;
}
}

private static Predicate<String> isChangedHeaders(List<String> headers) {
return h -> !headers.contains(h);
}

private static boolean isAerialOrUnderground(List<String> headers, OdreConfig odreConfig) {
return new HashSet<>(headers).containsAll(odreConfig.aerialLinesExpectedHeaders()) ||
new HashSet<>(headers).containsAll(odreConfig.undergroundLinesExpectedHeaders());
}

private static void getResultOrLogError(List<String> headers, List<String> expectedHeaders, Map<String, Reader> mapResult, String fileType, Path path) throws IOException {
if (new HashSet<>(headers).containsAll(expectedHeaders)) {
mapResult.putIfAbsent(fileType, new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8)));
if (expectedHeaders.stream().allMatch(headerMap::containsKey)) {
return true;
} else {
List<String> notFoundHeaders = expectedHeaders.stream().filter(isChangedHeaders(headers)).collect(Collectors.toList());
logHeaderError(path, notFoundHeaders);
List<String> notFoundHeaders = expectedHeaders.stream().filter(h -> !headerMap.containsKey(h)).toList();
LOGGER.error(HEADERS_OF_FILE_HAS_CHANGED, fileType, notFoundHeaders);
return false;
}
}

private static void logHeaderError(Path path, List<String> notFoundHeaders) {
LOGGER.error(HEADERS_OF_FILE_HAS_CHANGED, path.getFileName(), notFoundHeaders);
}
}

Loading

0 comments on commit f9a167d

Please sign in to comment.