From f9a167df4b958c45c7622566a4c0a48e0f33c30c Mon Sep 17 00:00:00 2001 From: Florian Dupuy <66690739+flo-dup@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:00:03 +0200 Subject: [PATCH] ODRE import postprocessor fixes (#3080) * Record LineGeoData * Reading substations once instead of twice * Do one graph traversal instead of three * Use SimpleGraph instead of PseudoGraph * Not calling twice containsVertex when creating graph * Avoid duplication of same coordinates * Only create graph if needed * Fix substations not filled * Not opening twice the file * Avoid list duplication * Remove LineGeoData/SubstationGeoData classes * Not reversing coordinates in case of equal distances Signed-off-by: Florian Dupuy --- .../iidm/geodata/elements/LineGeoData.java | 59 --- .../geodata/elements/SubstationGeoData.java | 38 -- .../iidm/geodata/odre/FileValidator.java | 141 ++----- .../geodata/odre/GeographicDataParser.java | 344 ++++++++++-------- .../iidm/geodata/odre/OdreGeoDataAdder.java | 11 +- .../odre/OdreGeoDataAdderPostProcessor.java | 6 +- .../geodata/odre/OdreGeoDataCsvLoader.java | 41 +-- .../geodata/utils/DistanceCalculator.java | 6 + .../iidm/geodata/utils/InputUtils.java | 10 +- .../utils/LineCoordinatesOrdering.java | 85 +++++ .../powsybl/iidm/geodata/utils/LineGraph.java | 8 +- .../utils/NetworkGeoDataExtensionsAdder.java | 34 +- .../geodata/elements/LineGeoDataTest.java | 34 -- .../elements/SubstationGeoDataTest.java | 28 -- .../iidm/geodata/odre/FileValidatorTest.java | 35 +- .../geodata/odre/OdreGeoDataAdderTest.java | 13 +- .../odre/OdreGeoDataCsvLoaderTest.java | 48 --- .../NetworkGeoDataExtensionsAdderTest.java | 12 +- .../valid-line-name/aerial-lines.csv | 2 - .../resources/valid-line-name/substations.csv | 3 - .../valid-line-name/underground-lines.csv | 2 - 21 files changed, 387 insertions(+), 573 deletions(-) delete mode 100644 iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java delete mode 100644 iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/SubstationGeoData.java create mode 100644 iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineCoordinatesOrdering.java delete mode 100644 iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/LineGeoDataTest.java delete mode 100644 iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/SubstationGeoDataTest.java delete mode 100644 iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoaderTest.java delete mode 100644 iidm/iidm-geodata/src/test/resources/valid-line-name/aerial-lines.csv delete mode 100644 iidm/iidm-geodata/src/test/resources/valid-line-name/substations.csv delete mode 100644 iidm/iidm-geodata/src/test/resources/valid-line-name/underground-lines.csv diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java deleted file mode 100644 index 489bdaae390..00000000000 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/LineGeoData.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (c) 2019, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.iidm.geodata.elements; - -import com.powsybl.iidm.network.extensions.Coordinate; - -import java.util.List; - -/** - * @author Geoffroy Jamgotchian {@literal } - * @author Chamseddine Benhamed {@literal } - */ -public class LineGeoData { - - private String id; - private String country1; - private String country2; - private String substationStart; - private String substationEnd; - private List coordinates; - - public LineGeoData(String id, String country1, String country2, String substationStart, String substationEnd, List coordinates) { - this.id = id; - this.country1 = country1; - this.country2 = country2; - this.substationStart = substationStart; - this.substationEnd = substationEnd; - this.coordinates = coordinates; - } - - public String getId() { - return id; - } - - public String getCountry1() { - return country1; - } - - public String getCountry2() { - return country2; - } - - public String getSubstationStart() { - return substationStart; - } - - public String getSubstationEnd() { - return substationEnd; - } - - public List getCoordinates() { - return coordinates; - } -} diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/SubstationGeoData.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/SubstationGeoData.java deleted file mode 100644 index b8d281d4c35..00000000000 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/elements/SubstationGeoData.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright (c) 2019, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.iidm.geodata.elements; - -import com.powsybl.iidm.network.extensions.Coordinate; - -/** - * @author Chamseddine Benhamed {@literal } - */ -public class SubstationGeoData { - - private String id; - private String country; - private Coordinate coordinate; - - public SubstationGeoData(String id, String country, Coordinate coordinate) { - this.id = id; - this.country = country; - this.coordinate = coordinate; - } - - public String getId() { - return id; - } - - public String getCountry() { - return country; - } - - public Coordinate getCoordinate() { - return coordinate; - } -} diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/FileValidator.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/FileValidator.java index cc4e34e3a8d..d4a5f54c930 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/FileValidator.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/FileValidator.java @@ -7,21 +7,13 @@ */ package com.powsybl.iidm.geodata.odre; -import com.powsybl.iidm.geodata.utils.InputUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.function.Predicate; -import java.util.stream.Collectors; +import java.util.List; +import java.util.Map; /** * @author Ahmed Bendaamer {@literal } @@ -32,133 +24,48 @@ public final class FileValidator { private FileValidator() { } - private static final String HEADERS_OF_FILE_HAS_CHANGED = "Invalid file, Headers of file {} has changed, header(s) not found: {}"; + private static final String HEADERS_OF_FILE_HAS_CHANGED = "Invalid {} file, header(s) not found: {}"; private static final Logger LOGGER = LoggerFactory.getLogger(FileValidator.class); static final String COUNTRY_FR = "FR"; - private static CSVFormat.Builder createCsvFormatBuilder() { - return CSVFormat.DEFAULT.builder() - .setQuote('"') - .setDelimiter(";") - .setRecordSeparator(System.lineSeparator()); - } - - static final CSVFormat CSV_FORMAT = createCsvFormatBuilder() + static final CSVFormat CSV_FORMAT = CSVFormat.DEFAULT.builder() + .setQuote('"') + .setDelimiter(";") + .setRecordSeparator(System.lineSeparator()) .setHeader() .setSkipHeaderRecord(true) .build(); - static final CSVFormat CSV_FORMAT_FOR_HEADER = createCsvFormatBuilder().build(); + public static final String SUBSTATIONS = "substations"; public static final String AERIAL_LINES = "aerial-lines"; public static final String UNDERGROUND_LINES = "underground-lines"; - public static boolean validateSubstations(Path path, OdreConfig odreConfig) { - try (Reader reader = new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))) { - Iterator records = CSVParser.parse(reader, FileValidator.CSV_FORMAT_FOR_HEADER).iterator(); - - List headers; - if (records.hasNext()) { - CSVRecord headersRecord = records.next(); - headers = headersRecord.toList(); - } else { - LOGGER.error("The file {} is empty", path.getFileName()); - return false; - } - - if (new HashSet<>(headers).containsAll(odreConfig.substationsExpectedHeaders())) { - return true; - } else { - List notFoundHeaders = odreConfig.substationsExpectedHeaders().stream().filter(isChangedHeaders(headers)).collect(Collectors.toList()); - logHeaderError(path, notFoundHeaders); - } - } catch (IOException e) { - LOGGER.error(e.getMessage()); - return false; - } - return false; + public static boolean validateSubstationsHeaders(CSVParser records, OdreConfig odreConfig) { + return validateHeaders(records, odreConfig.substationsExpectedHeaders(), FileValidator.SUBSTATIONS); } - public static Map validateLines(List paths, OdreConfig odreConfig) { - Map mapResult = new HashMap<>(); - paths.forEach(path -> { - try (Reader reader = new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))) { - Iterator records = CSVParser.parse(reader, FileValidator.CSV_FORMAT_FOR_HEADER).iterator(); - - final List headers; - if (records.hasNext()) { - CSVRecord headersRecord = records.next(); - headers = headersRecord.toList(); - } else { - headers = null; - LOGGER.error("The file {} is empty", path.getFileName()); - } - - String equipmentType = null; - if (headers != null && records.hasNext()) { - CSVRecord firstRow = records.next(); - equipmentType = readEquipmentType(firstRow, headers, odreConfig); - } else { - LOGGER.error("The file {} has no data", path.getFileName()); - } - - String type = equipmentType != null ? equipmentType : odreConfig.nullEquipmentType(); - if (type.equals(odreConfig.nullEquipmentType())) { - getIfSubstationsOrLogError(mapResult, path, headers, equipmentType, odreConfig); - } else if (type.equals(odreConfig.aerialEquipmentType())) { - getResultOrLogError(headers, odreConfig.aerialLinesExpectedHeaders(), mapResult, AERIAL_LINES, path); - } else if (type.equals(odreConfig.undergroundEquipmentType())) { - getResultOrLogError(headers, odreConfig.undergroundLinesExpectedHeaders(), mapResult, UNDERGROUND_LINES, path); - } else { - LOGGER.error("The file {} has no known equipment type : {}", path.getFileName(), equipmentType); - } - } catch (IOException e) { - mapResult.values().forEach(IOUtils::closeQuietly); - throw new UncheckedIOException(e); - } - }); - return mapResult; + public static boolean validateAerialLinesHeaders(CSVParser records, OdreConfig odreConfig) { + return validateHeaders(records, odreConfig.aerialLinesExpectedHeaders(), FileValidator.AERIAL_LINES); } - private static String readEquipmentType(CSVRecord firstRow, List headers, OdreConfig odreConfig) { - String equipmentType = null; - int index = headers.indexOf(odreConfig.equipmentTypeColumn()); - if (index != -1) { - equipmentType = firstRow.get(index); - } - return equipmentType; + public static boolean validateUndergroundHeaders(CSVParser records, OdreConfig odreConfig) { + return validateHeaders(records, odreConfig.undergroundLinesExpectedHeaders(), FileValidator.UNDERGROUND_LINES); } - private static void getIfSubstationsOrLogError(Map mapResult, Path path, List headers, String typeOuvrage, OdreConfig odreConfig) throws IOException { - if (new HashSet<>(headers).containsAll(odreConfig.substationsExpectedHeaders())) { - mapResult.putIfAbsent(SUBSTATIONS, new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))); - } else if (isAerialOrUnderground(headers, odreConfig)) { - LOGGER.error("The file {} has no equipment type : {}", path.getFileName(), typeOuvrage); - } else { - List notFoundHeaders = odreConfig.substationsExpectedHeaders().stream().filter(isChangedHeaders(headers)).collect(Collectors.toList()); - logHeaderError(path, notFoundHeaders); + private static boolean validateHeaders(CSVParser csvParser, List expectedHeaders, String fileType) { + Map headerMap = csvParser.getHeaderMap(); + if (headerMap.isEmpty()) { + LOGGER.error("The substations file is empty"); + return false; } - } - - private static Predicate isChangedHeaders(List headers) { - return h -> !headers.contains(h); - } - private static boolean isAerialOrUnderground(List headers, OdreConfig odreConfig) { - return new HashSet<>(headers).containsAll(odreConfig.aerialLinesExpectedHeaders()) || - new HashSet<>(headers).containsAll(odreConfig.undergroundLinesExpectedHeaders()); - } - - private static void getResultOrLogError(List headers, List expectedHeaders, Map mapResult, String fileType, Path path) throws IOException { - if (new HashSet<>(headers).containsAll(expectedHeaders)) { - mapResult.putIfAbsent(fileType, new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8))); + if (expectedHeaders.stream().allMatch(headerMap::containsKey)) { + return true; } else { - List notFoundHeaders = expectedHeaders.stream().filter(isChangedHeaders(headers)).collect(Collectors.toList()); - logHeaderError(path, notFoundHeaders); + List notFoundHeaders = expectedHeaders.stream().filter(h -> !headerMap.containsKey(h)).toList(); + LOGGER.error(HEADERS_OF_FILE_HAS_CHANGED, fileType, notFoundHeaders); + return false; } } - - private static void logHeaderError(Path path, List notFoundHeaders) { - LOGGER.error(HEADERS_OF_FILE_HAS_CHANGED, path.getFileName(), notFoundHeaders); - } } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/GeographicDataParser.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/GeographicDataParser.java index 4046d177457..1acd771a7ef 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/GeographicDataParser.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/GeographicDataParser.java @@ -7,10 +7,8 @@ */ package com.powsybl.iidm.geodata.odre; -import com.google.common.collect.Lists; +import com.fasterxml.jackson.core.JsonProcessingException; import com.powsybl.iidm.geodata.elements.GeoShape; -import com.powsybl.iidm.geodata.elements.LineGeoData; -import com.powsybl.iidm.geodata.elements.SubstationGeoData; import com.powsybl.iidm.geodata.utils.DistanceCalculator; import com.powsybl.iidm.geodata.utils.GeoShapeDeserializer; import com.powsybl.iidm.geodata.utils.LineGraph; @@ -18,10 +16,10 @@ import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.time.StopWatch; -import org.apache.commons.lang3.tuple.Pair; import org.jgrapht.Graph; -import org.jgrapht.Graphs; -import org.jgrapht.alg.connectivity.ConnectivityInspector; +import org.jgrapht.event.ConnectedComponentTraversalEvent; +import org.jgrapht.event.TraversalListenerAdapter; +import org.jgrapht.event.VertexTraversalEvent; import org.jgrapht.traverse.BreadthFirstIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,66 +45,83 @@ private GeographicDataParser() { private static final Logger LOGGER = LoggerFactory.getLogger(GeographicDataParser.class); private static final int THRESHOLD = 5; - public static Map parseSubstations(Reader reader, OdreConfig odreConfig) { - Map substations = new HashMap<>(); + /** + * Parse the substations CSV data contained in the given reader, using the given odreConfig for column names. + * @return the map of substation coordinates indexed by the substation id + */ + public static Map parseSubstations(Reader reader, OdreConfig odreConfig) { + Map substations = new LinkedHashMap<>(); StopWatch stopWatch = new StopWatch(); stopWatch.start(); - int substationCount = 0; try { - Iterable records = CSVParser.parse(reader, FileValidator.CSV_FORMAT); - for (CSVRecord row : records) { - String id = row.get(odreConfig.substationIdColumn()); - double lon = Double.parseDouble(row.get(odreConfig.substationLongitudeColumn())); - double lat = Double.parseDouble(row.get(odreConfig.substationLatitudeColumn())); - SubstationGeoData substation = substations.get(id); - if (substation == null) { - SubstationGeoData substationGeoData = new SubstationGeoData(id, FileValidator.COUNTRY_FR, new Coordinate(lat, lon)); - substations.put(id, substationGeoData); + CSVParser records = CSVParser.parse(reader, FileValidator.CSV_FORMAT); + if (FileValidator.validateSubstationsHeaders(records, odreConfig)) { + for (CSVRecord row : records) { + String id = row.get(odreConfig.substationIdColumn()); + substations.computeIfAbsent(id, key -> { + double lon = Double.parseDouble(row.get(odreConfig.substationLongitudeColumn())); + double lat = Double.parseDouble(row.get(odreConfig.substationLatitudeColumn())); + return new Coordinate(lat, lon); + }); } - substationCount++; } } catch (IOException e) { throw new UncheckedIOException(e); } - LOGGER.info("{} substations read in {} ms", substationCount, stopWatch.getTime()); + LOGGER.info("{} substations read in {} ms", substations.size(), stopWatch.getTime()); return substations; } - private static double distanceCoordinate(Coordinate coord1, Coordinate coord2) { - return DistanceCalculator.distance(coord1.getLatitude(), coord1.getLongitude(), coord2.getLatitude(), coord2.getLongitude()); - } + /** + * Parse the lines CSV data contained in the given readers, using the given odreConfig for column names and line types. + * @param aerialLinesReader the reader containing the aerial lines CSV data + * @param undergroundLinesReader the reader containing the underground lines CSV data + * @param odreConfig the config used for column names and line types + * @return the map of line coordinates indexed by the line id + */ + public static Map> parseLines(Reader aerialLinesReader, Reader undergroundLinesReader, OdreConfig odreConfig) { + try { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); - public static Pair substationOrder(Map substationGeoData, String lineId, List coordinates) { - String substation1 = lineId.substring(0, 5).trim(); - String substation2 = lineId.substring(8).trim(); - SubstationGeoData geo1 = substationGeoData.get(substation1); - SubstationGeoData geo2 = substationGeoData.get(substation2); - - if (geo1 == null && geo2 == null) { - LOGGER.warn("can't find any substation for {}", lineId); - return Pair.of("", ""); - } else if (geo1 != null && geo2 != null) { - return findStartAndEndSubstationsOfLine(lineId, geo1, geo2, substation1, substation2, - coordinates.get(0), coordinates.get(coordinates.size() - 1)); - } else { - boolean isStart = distanceCoordinate((geo1 != null ? geo1 : geo2).getCoordinate(), coordinates.get(0)) < distanceCoordinate((geo1 != null ? geo1 : geo2).getCoordinate(), coordinates.get(coordinates.size() - 1)); - String substation = geo1 != null ? substation1 : substation2; - return Pair.of(isStart ? substation : "", isStart ? "" : substation); - } - } + Map>> coordinatesListsByLine = new HashMap<>(); - public static Map parseLines(Reader aerialLinesReader, Reader undergroundLinesReader, - Map stringSubstationGeoDataMap, OdreConfig odreConfig) { - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); + CSVParser aerialLinesRecords = CSVParser.parse(aerialLinesReader, FileValidator.CSV_FORMAT); + CSVParser undergroundLinesRecords = CSVParser.parse(undergroundLinesReader, FileValidator.CSV_FORMAT); - Map> graphByLine = new HashMap<>(); + if (!FileValidator.validateAerialLinesHeaders(aerialLinesRecords, odreConfig) + || !FileValidator.validateUndergroundHeaders(undergroundLinesRecords, odreConfig)) { + return Collections.emptyMap(); + } - parseLine(graphByLine, aerialLinesReader, odreConfig); - parseLine(graphByLine, undergroundLinesReader, odreConfig); + parseLine(coordinatesListsByLine, aerialLinesRecords, odreConfig); + parseLine(coordinatesListsByLine, undergroundLinesRecords, odreConfig); + + Map> lines = fixLines(coordinatesListsByLine); + + LOGGER.info("{} lines read in {} ms", lines.size(), stopWatch.getTime()); + + if (coordinatesListsByLine.size() != lines.size()) { + LOGGER.warn("Total discarded lines : {}/{} ", + coordinatesListsByLine.size() - lines.size(), coordinatesListsByLine.size()); + } + + return lines; + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - Map lines = new HashMap<>(); + /** + * "Fixing" the lines coordinates data: this method tries to calculate a single list when there are several lists for + * a given line, which often occurs in the ODRE data. + * @param coordinatesListsByLine the map of all the lists of coordinates, indexed by the line id + * @return the map of the "fixed" line coordinates, indexed by the line id + */ + private static Map> fixLines(Map>> coordinatesListsByLine) { + Map> lines = new HashMap<>(); int linesWithOneConnectedSet = 0; int linesWithTwoOrMoreConnectedSets = 0; @@ -114,120 +129,123 @@ public static Map parseLines(Reader aerialLinesReader, Read int oneConnectedSetDiscarded = 0; int twoOrMoreConnectedSetsDiscarded = 0; - for (Map.Entry> e : graphByLine.entrySet()) { + for (Map.Entry>> e : coordinatesListsByLine.entrySet()) { String lineId = e.getKey(); - Graph graph = e.getValue(); - List> connectedSets = new ConnectivityInspector<>(graph).connectedSets(); - if (connectedSets.size() == 1) { - linesWithOneConnectedSet++; - List ends = getEnds(connectedSets.get(0), graph); - if (ends.size() == 2) { - List coordinates = Lists.newArrayList(new BreadthFirstIterator<>(graph, ends.get(0))); - Pair substations = substationOrder(stringSubstationGeoDataMap, lineId, coordinates); - LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), coordinates); - lines.put(lineId, line); - } else { - oneConnectedSetDiscarded++; - } + + List> coordinatesLists = e.getValue(); + if (coordinatesLists.size() == 1) { + // Easy case: only one list + lines.put(lineId, coordinatesLists.get(0)); } else { - linesWithTwoOrMoreConnectedSets++; - List> coordinatesComponents = fillMultipleConnectedSetsCoordinatesList(connectedSets, - graph); + // We want to calculate a single list: we construct a graph based on the coordinates, adding a vertex + // for each coordinate, and adding an edge between two consecutive coordinates + LineGraph graph = new LineGraph<>(Object.class); + coordinatesLists.forEach(graph::addVerticesAndEdges); + List connectedSets = getConnectedSets(graph); + if (connectedSets.size() == 1) { + linesWithOneConnectedSet++; + ConnectedSet connectedSet = connectedSets.get(0); + if (connectedSet.ends().size() == 2) { + // Only one connected set and two ends: this is a single line + lines.put(lineId, connectedSet.list()); + } else { + // Only one connected set and more than two ends: there are forks in the lines + oneConnectedSetDiscarded++; + } + } else { - if (coordinatesComponents.size() != connectedSets.size()) { - twoOrMoreConnectedSetsDiscarded++; - continue; - } + // there are several connected sets: we try to order them to have a single line + linesWithTwoOrMoreConnectedSets++; + List> coordinatesComponents = createMultipleConnectedSetsCoordinatesList(connectedSets); - List aggregatedCoordinates = aggregateCoordinates(coordinatesComponents); - Pair substations = substationOrder(stringSubstationGeoDataMap, lineId, aggregatedCoordinates); - LineGeoData line = new LineGeoData(lineId, FileValidator.COUNTRY_FR, FileValidator.COUNTRY_FR, substations.getLeft(), substations.getRight(), aggregatedCoordinates); - lines.put(lineId, line); + if (coordinatesComponents.size() != connectedSets.size() || coordinatesComponents.size() > 2) { + // This happens if there is a fork in one of the connected components or if there are more than 2 connected components + twoOrMoreConnectedSetsDiscarded++; + continue; + } + + lines.put(lineId, aggregateCoordinates(coordinatesComponents)); + } } } - - LOGGER.info("{} lines read in {} ms", lines.size(), stopWatch.getTime()); LOGGER.info("{} lines have one Connected set, {} of them were discarded", linesWithOneConnectedSet, oneConnectedSetDiscarded); LOGGER.info("{} lines have two or more Connected sets, {} of them were discarded", linesWithTwoOrMoreConnectedSets, twoOrMoreConnectedSetsDiscarded); - - if (graphByLine.size() != lines.size()) { - LOGGER.warn("Total discarded lines : {}/{} ", - graphByLine.size() - lines.size(), graphByLine.size()); - } - return lines; } - private static void parseLine(Map> graphByLine, Reader reader, OdreConfig odreConfig) { - try { - Iterable records = CSVParser.parse(reader, FileValidator.CSV_FORMAT); - Map idsColumnNames = odreConfig.idsColumnNames(); - for (CSVRecord row : records) { - List ids = Stream.of(row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_1)), - row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_2)), - row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_3)), - row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_4)), - row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_5))).filter(Objects::nonNull).toList(); - GeoShape geoShape = GeoShapeDeserializer.read(row.get(odreConfig.geoShapeColumn())); - - if (ids.isEmpty() || geoShape.coordinates().isEmpty()) { - continue; - } - - for (String lineId : ids) { - putLineGraph(lineId, graphByLine).addVerticesAndEdges(geoShape.coordinates()); - } + /** + * Compute the connected sets for the given graph. The corresponding list of coordinates and list of extremities + * for each connected component is computed at the same time. + */ + private static List getConnectedSets(Graph graph) { + List connectedSets = new ArrayList<>(); + Set vertexSet = graph.vertexSet(); + + Optional endCoord = vertexSet.stream().filter(v -> graph.degreeOf(v) == 1).findFirst(); + if (endCoord.isPresent()) { + var bfi = new BreadthFirstIterator(graph, () -> Stream.concat(Stream.of(endCoord.get()), vertexSet.stream()).iterator()); + bfi.addTraversalListener(new GeoTraversalListener(graph, connectedSets)); + while (bfi.hasNext()) { + bfi.next(); } - } catch (IOException e) { - throw new UncheckedIOException(e); + } else { + connectedSets = List.of(new ConnectedSet(vertexSet.stream().toList(), List.of())); } + return connectedSets; } - private static LineGraph putLineGraph(String lineId, Map> graphByLine) { - return (LineGraph) graphByLine.computeIfAbsent(lineId, key -> new LineGraph<>(Object.class)); - } + /** + * Parsing the CSV lines data which is given by the CSVParser, using the given odreConfig for column names and line types. + * The resulting coordinates are put in the given map. + */ + private static void parseLine(Map>> coordinateListsByLine, CSVParser csvParser, OdreConfig odreConfig) throws JsonProcessingException { + Map idsColumnNames = odreConfig.idsColumnNames(); + for (CSVRecord row : csvParser) { + List ids = Stream.of(row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_1)), + row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_2)), + row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_3)), + row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_4)), + row.get(idsColumnNames.get(OdreConfig.LINE_ID_KEY_5))).filter(Objects::nonNull).filter(s -> !s.isEmpty()).distinct().toList(); + GeoShape geoShape = GeoShapeDeserializer.read(row.get(odreConfig.geoShapeColumn())); + + if (ids.isEmpty() || geoShape.coordinates().isEmpty()) { + continue; + } - private static List getEnds(Set connectedSet, Graph graph) { - List ends = new ArrayList<>(); - for (Coordinate coordinate : connectedSet) { - if (Graphs.neighborListOf(graph, coordinate).size() == 1) { - ends.add(coordinate); + for (String lineId : ids) { + coordinateListsByLine.computeIfAbsent(lineId, key -> new ArrayList<>()) + .add(geoShape.coordinates()); } } - return ends; } + /** + * Calculate the distance between the first and the last coordinates of the given list + */ private static double getBranchLength(List coordinatesComponent) { - return DistanceCalculator.distance(coordinatesComponent.get(0).getLatitude(), coordinatesComponent.get(0).getLongitude(), - coordinatesComponent.get(coordinatesComponent.size() - 1).getLatitude(), coordinatesComponent.get(coordinatesComponent.size() - 1).getLongitude()); + return DistanceCalculator.distance(coordinatesComponent.get(0), coordinatesComponent.get(coordinatesComponent.size() - 1)); } + /** + * Aggregate coordinates of two connected components into one single list by trying to find which extremity connects to which other extremity + */ private static List aggregateCoordinates(List> coordinatesComponents) { - coordinatesComponents.sort((comp1, comp2) -> (int) (getBranchLength(comp2) - getBranchLength(comp1))); - return aggregateCoordinates(coordinatesComponents.get(0), coordinatesComponents.get(1)); - } - - private static List aggregateCoordinates(List coordinatesComponent1, List coordinatesComponent2) { List aggregatedCoordinates; + List coordinatesComponent1 = coordinatesComponents.get(0); + List coordinatesComponent2 = coordinatesComponents.get(1); + double l1 = getBranchLength(coordinatesComponent1); double l2 = getBranchLength(coordinatesComponent2); - if (100 * l2 / l1 < THRESHOLD) { - return coordinatesComponent1; + if (100 * Math.min(l1, l2) / Math.max(l1, l2) < THRESHOLD) { + return l1 > l2 ? coordinatesComponent1 : coordinatesComponent2; } - double d1 = DistanceCalculator.distance(coordinatesComponent1.get(0).getLatitude(), coordinatesComponent1.get(0).getLongitude(), - coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLatitude(), coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLongitude()); - - double d2 = DistanceCalculator.distance(coordinatesComponent1.get(0).getLatitude(), coordinatesComponent1.get(0).getLongitude(), - coordinatesComponent2.get(0).getLatitude(), coordinatesComponent2.get(0).getLongitude()); - - double d3 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLatitude(), coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLongitude(), - coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLatitude(), coordinatesComponent2.get(coordinatesComponent2.size() - 1).getLongitude()); - - double d4 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLatitude(), coordinatesComponent1.get(coordinatesComponent1.size() - 1).getLongitude(), - coordinatesComponent2.get(0).getLatitude(), coordinatesComponent2.get(0).getLongitude()); + double d1 = DistanceCalculator.distance(coordinatesComponent1.get(0), coordinatesComponent2.get(coordinatesComponent2.size() - 1)); + double d2 = DistanceCalculator.distance(coordinatesComponent1.get(0), coordinatesComponent2.get(0)); + double d3 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1), coordinatesComponent2.get(coordinatesComponent2.size() - 1)); + double d4 = DistanceCalculator.distance(coordinatesComponent1.get(coordinatesComponent1.size() - 1), coordinatesComponent2.get(0)); List distances = Arrays.asList(d1, d2, d3, d4); double min = min(distances); @@ -252,32 +270,54 @@ private static List aggregateCoordinates(List coordinate return aggregatedCoordinates; } - private static Pair findStartAndEndSubstationsOfLine(String lineId, SubstationGeoData geo1, SubstationGeoData geo2, - String substation1, String substation2, - Coordinate firstCoordinate, Coordinate lastCoordinate) { - final double sub1pil1 = distanceCoordinate(geo1.getCoordinate(), firstCoordinate); - final double sub2pil1 = distanceCoordinate(geo2.getCoordinate(), firstCoordinate); - final double sub1pil2 = distanceCoordinate(geo1.getCoordinate(), lastCoordinate); - final double sub2pil2 = distanceCoordinate(geo2.getCoordinate(), lastCoordinate); - if ((sub1pil1 < sub2pil1) == (sub1pil2 < sub2pil2)) { - LOGGER.error("line {} for substations {} and {} has both first and last coordinate nearest to {}", lineId, substation1, substation2, sub1pil1 < sub2pil1 ? substation1 : substation2); - return Pair.of("", ""); - } - return Pair.of(sub1pil1 < sub2pil1 ? substation1 : substation2, sub1pil1 < sub2pil1 ? substation2 : substation1); - } - - private static List> fillMultipleConnectedSetsCoordinatesList(List> connectedSets, - Graph graph) { + /** + * Constructing the list of coordinates of each connected component, filtering out the connected sets which have more than two ends (or zero) + */ + private static List> createMultipleConnectedSetsCoordinatesList(List connectedSets) { List> coordinatesComponents = new ArrayList<>(); - for (Set connectedSet : connectedSets) { - List endsComponent = getEnds(connectedSet, graph); + for (ConnectedSet connectedSet : connectedSets) { + List endsComponent = connectedSet.ends(); if (endsComponent.size() == 2) { - List coordinatesComponent = Lists.newArrayList(new BreadthFirstIterator<>(graph, endsComponent.get(0))); - coordinatesComponents.add(coordinatesComponent); + coordinatesComponents.add(connectedSet.list()); } else { break; } } return coordinatesComponents; } + + private record ConnectedSet(List list, List ends) { + } + + private static class GeoTraversalListener extends TraversalListenerAdapter { + private final List connectedSets; + private final Graph graph; + private List currentConnectedSet; + private List currentConnectedSetEnds; + + public GeoTraversalListener(Graph graph, List connectedSets) { + this.graph = graph; + this.connectedSets = connectedSets; + } + + @Override + public void connectedComponentFinished(ConnectedComponentTraversalEvent e) { + connectedSets.add(new ConnectedSet(currentConnectedSet, currentConnectedSetEnds)); + } + + @Override + public void connectedComponentStarted(ConnectedComponentTraversalEvent e) { + currentConnectedSet = new ArrayList<>(); + currentConnectedSetEnds = new ArrayList<>(); + } + + @Override + public void vertexTraversed(VertexTraversalEvent e) { + Coordinate v = e.getVertex(); + currentConnectedSet.add(v); + if (graph.degreeOf(v) == 1) { + currentConnectedSetEnds.add(v); + } + } + } } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdder.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdder.java index 345c2ca9775..9f6f5ba1746 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdder.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdder.java @@ -9,6 +9,7 @@ import com.powsybl.iidm.network.Network; +import java.io.IOException; import java.nio.file.Path; import static com.powsybl.iidm.geodata.utils.NetworkGeoDataExtensionsAdder.fillNetworkLinesGeoData; @@ -22,13 +23,11 @@ public class OdreGeoDataAdder { protected OdreGeoDataAdder() { } - public static void fillNetworkSubstationsGeoDataFromFile(Network network, Path path, OdreConfig odreConfig) { - fillNetworkSubstationsGeoData(network, OdreGeoDataCsvLoader.getSubstationsGeoData(path, odreConfig)); + public static void fillNetworkSubstationsGeoDataFromFile(Network network, Path path, OdreConfig odreConfig) throws IOException { + fillNetworkSubstationsGeoData(network, OdreGeoDataCsvLoader.getSubstationsCoordinates(path, odreConfig)); } - public static void fillNetworkLinesGeoDataFromFiles(Network network, Path aerialLinesFilePath, - Path undergroundLinesFilePath, Path substationPath, OdreConfig odreConfig) { - fillNetworkLinesGeoData(network, - OdreGeoDataCsvLoader.getLinesGeoData(aerialLinesFilePath, undergroundLinesFilePath, substationPath, odreConfig)); + public static void fillNetworkLinesGeoDataFromFiles(Network network, Path aerialLinesFilePath, Path undergroundLinesFilePath, OdreConfig odreConfig) throws IOException { + fillNetworkLinesGeoData(network, OdreGeoDataCsvLoader.getLinesCoordinates(aerialLinesFilePath, undergroundLinesFilePath, odreConfig)); } } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderPostProcessor.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderPostProcessor.java index 95b62fdd263..e109bd0573a 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderPostProcessor.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderPostProcessor.java @@ -16,6 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -76,14 +77,13 @@ public String getName() { } @Override - public void process(Network network, ComputationManager computationManager) { + public void process(Network network, ComputationManager computationManager) throws IOException { if (Files.exists(substationsFilePath)) { OdreGeoDataAdder.fillNetworkSubstationsGeoDataFromFile(network, substationsFilePath, odreConfig); boolean aerialLinesPresent = Files.exists(aerialLinesFilePath); boolean undergroundLinesPresent = Files.exists(undergroundLinesFilePath); if (aerialLinesPresent && undergroundLinesPresent) { - OdreGeoDataAdder.fillNetworkLinesGeoDataFromFiles(network, - aerialLinesFilePath, undergroundLinesFilePath, substationsFilePath, odreConfig); + OdreGeoDataAdder.fillNetworkLinesGeoDataFromFiles(network, aerialLinesFilePath, undergroundLinesFilePath, odreConfig); } else { String missingAerialFiles = aerialLinesPresent ? "" : aerialLinesFilePath + " "; String missingFiles = missingAerialFiles.concat(undergroundLinesPresent ? "" : undergroundLinesFilePath.toString()); diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoader.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoader.java index 3fb654da37f..369c1bb55cb 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoader.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoader.java @@ -7,16 +7,12 @@ */ package com.powsybl.iidm.geodata.odre; -import com.powsybl.iidm.geodata.elements.LineGeoData; -import com.powsybl.iidm.geodata.elements.SubstationGeoData; import com.powsybl.iidm.geodata.utils.InputUtils; -import org.apache.commons.io.IOUtils; +import com.powsybl.iidm.network.extensions.Coordinate; -import java.io.*; -import java.nio.file.Files; +import java.io.IOException; +import java.io.Reader; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -28,32 +24,17 @@ public class OdreGeoDataCsvLoader { protected OdreGeoDataCsvLoader() { } - public static List getSubstationsGeoData(Path path, OdreConfig odreConfig) { - try (Reader reader = new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path))))) { - if (FileValidator.validateSubstations(path, odreConfig)) { - return new ArrayList<>(GeographicDataParser.parseSubstations(reader, odreConfig).values()); - } else { - return Collections.emptyList(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + public static Map getSubstationsCoordinates(Path path, OdreConfig odreConfig) throws IOException { + try (Reader reader = InputUtils.toReader(path)) { + return GeographicDataParser.parseSubstations(reader, odreConfig); } } - public static List getLinesGeoData(Path aerialLinesFilePath, Path undergroundLinesFilePath, - Path substationPath, OdreConfig odreConfig) { - Map mapValidation = FileValidator.validateLines(List.of(substationPath, - aerialLinesFilePath, undergroundLinesFilePath), odreConfig); - List result = Collections.emptyList(); - try { - if (mapValidation.size() == 3) { - result = new ArrayList<>(GeographicDataParser.parseLines(mapValidation.get(FileValidator.AERIAL_LINES), - mapValidation.get(FileValidator.UNDERGROUND_LINES), - GeographicDataParser.parseSubstations(mapValidation.get(FileValidator.SUBSTATIONS), odreConfig), odreConfig).values()); - } - } finally { - mapValidation.values().forEach(IOUtils::closeQuietly); + public static Map> getLinesCoordinates(Path aerialLinesFilePath, Path undergroundLinesFilePath, OdreConfig odreConfig) throws IOException { + try (Reader aerialReader = InputUtils.toReader(aerialLinesFilePath); + Reader undergroundReader = InputUtils.toReader(undergroundLinesFilePath)) { + + return GeographicDataParser.parseLines(aerialReader, undergroundReader, odreConfig); } - return result; } } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/DistanceCalculator.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/DistanceCalculator.java index 570df51d25d..4292bc1d3b7 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/DistanceCalculator.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/DistanceCalculator.java @@ -7,6 +7,8 @@ */ package com.powsybl.iidm.geodata.utils; +import com.powsybl.iidm.network.extensions.Coordinate; + /** * @author Chamseddine Benhamed {@literal } */ @@ -16,6 +18,10 @@ private DistanceCalculator() { } + public static double distance(Coordinate coord1, Coordinate coord2) { + return DistanceCalculator.distance(coord1.getLatitude(), coord1.getLongitude(), coord2.getLatitude(), coord2.getLongitude()); + } + /** * Compute an approximate distance in meters between two geographical points (latitude, longitude in degrees). * The computation assumes that the earth is spherical and its radius is equal to 6378137 meters. diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/InputUtils.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/InputUtils.java index 41669e4cdf0..0bee8bfeb77 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/InputUtils.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/InputUtils.java @@ -10,8 +10,10 @@ import org.apache.commons.io.ByteOrderMark; import org.apache.commons.io.input.BOMInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; /** * @author Hugo Marcellin {@literal } @@ -20,6 +22,10 @@ public final class InputUtils { private InputUtils() { } + public static Reader toReader(Path path) throws IOException { + return new BufferedReader(new InputStreamReader(InputUtils.toBomInputStream(Files.newInputStream(path)), StandardCharsets.UTF_8)); + } + public static BOMInputStream toBomInputStream(InputStream inputStream) throws IOException { return BOMInputStream.builder().setInputStream(inputStream).setByteOrderMarks(ByteOrderMark.UTF_8).get(); } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineCoordinatesOrdering.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineCoordinatesOrdering.java new file mode 100644 index 00000000000..2933a565b7d --- /dev/null +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineCoordinatesOrdering.java @@ -0,0 +1,85 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.iidm.geodata.utils; + +import com.google.common.collect.Lists; +import com.powsybl.iidm.network.Line; +import com.powsybl.iidm.network.TwoSides; +import com.powsybl.iidm.network.extensions.Coordinate; +import com.powsybl.iidm.network.extensions.SubstationPosition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; + +/** + * @author Florian Dupuy {@literal } + */ +public final class LineCoordinatesOrdering { + + private static final Logger LOGGER = LoggerFactory.getLogger(LineCoordinatesOrdering.class); + + private LineCoordinatesOrdering() { + + } + + public static List order(Line line, List lineCoordinates) { + + Optional s1 = getCoordinate(line, TwoSides.ONE); + Optional s2 = getCoordinate(line, TwoSides.TWO); + Coordinate firstCoordinate = lineCoordinates.get(0); + Coordinate lastCoordinate = lineCoordinates.get(lineCoordinates.size() - 1); + + if (s1.isPresent()) { + if (s2.isPresent()) { + return orderCoordinates(s1.get(), s2.get(), firstCoordinate, lastCoordinate, lineCoordinates, line.getId()); + } else { + return orderCoordinates(s1.get(), firstCoordinate, lastCoordinate, lineCoordinates); + } + } else { + if (s2.isPresent()) { + return orderCoordinates(s2.get(), lastCoordinate, firstCoordinate, lineCoordinates); + } else { + return lineCoordinates; // no information on substations positions -> considering it's in the right order + } + } + } + + private static List orderCoordinates(Coordinate s1, Coordinate s2, Coordinate firstCoordinate, Coordinate lastCoordinate, + List lineCoordinates, String lineId) { + final double substation1lineStart = DistanceCalculator.distance(s1, firstCoordinate); + final double substation2lineStart = DistanceCalculator.distance(s2, firstCoordinate); + final double substation1lineEnd = DistanceCalculator.distance(s1, lastCoordinate); + final double substation2lineEnd = DistanceCalculator.distance(s2, lastCoordinate); + if ((substation1lineStart < substation2lineStart) == (substation1lineEnd < substation2lineEnd)) { + LOGGER.warn("line {} has both first and last coordinate nearest to {}", + lineId, substation1lineStart < substation2lineStart ? TwoSides.ONE : TwoSides.TWO); + // reverse the list if line end closer to substation1 than line start + return substation1lineStart <= substation1lineEnd ? lineCoordinates : Lists.reverse(lineCoordinates); + } + // main case: the line start is closer to one substation, the line end is closer to the other substation + // reverse the list if the start is closer to the second substation + return substation1lineStart <= substation2lineStart ? lineCoordinates : Lists.reverse(lineCoordinates); + } + + private static List orderCoordinates(Coordinate substation, Coordinate coordA, Coordinate coordB, List coordinates) { + if (DistanceCalculator.distance(substation, coordA) > DistanceCalculator.distance(substation, coordB)) { + return Lists.reverse(coordinates); + } else { + return coordinates; + } + } + + private static Optional getCoordinate(Line line, TwoSides side) { + return line.getTerminal(side).getVoltageLevel().getSubstation() + .map(s -> s.getExtension(SubstationPosition.class)) + .map(SubstationPosition.class::cast) + .map(SubstationPosition::getCoordinate); + } +} diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineGraph.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineGraph.java index 2059bcbb521..b25ec6cc36a 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineGraph.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/LineGraph.java @@ -7,12 +7,12 @@ */ package com.powsybl.iidm.geodata.utils; -import org.jgrapht.graph.Pseudograph; +import org.jgrapht.graph.SimpleGraph; /** * @author Hugo Marcellin {@literal } */ -public class LineGraph extends Pseudograph { +public class LineGraph extends SimpleGraph { public LineGraph(Class edgeClass) { super(edgeClass); } @@ -20,9 +20,7 @@ public LineGraph(Class edgeClass) { public void addVerticesAndEdges(Iterable vertices) { V previousVertex = null; for (V vertex : vertices) { - if (!containsVertex(vertex)) { - addVertex(vertex); - } + addVertex(vertex); if (previousVertex != null) { addEdge(previousVertex, vertex); } diff --git a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdder.java b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdder.java index e17dda60498..7e22a9f2f51 100644 --- a/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdder.java +++ b/iidm/iidm-geodata/src/main/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdder.java @@ -7,44 +7,52 @@ */ package com.powsybl.iidm.geodata.utils; -import com.powsybl.iidm.geodata.elements.LineGeoData; -import com.powsybl.iidm.geodata.elements.SubstationGeoData; import com.powsybl.iidm.network.Line; import com.powsybl.iidm.network.Network; import com.powsybl.iidm.network.Substation; +import com.powsybl.iidm.network.extensions.Coordinate; import com.powsybl.iidm.network.extensions.LinePositionAdder; import com.powsybl.iidm.network.extensions.SubstationPositionAdder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; /** * @author Hugo Kulesza {@literal } */ public final class NetworkGeoDataExtensionsAdder { + private static final Logger LOGGER = LoggerFactory.getLogger(NetworkGeoDataExtensionsAdder.class); + private NetworkGeoDataExtensionsAdder() { } - public static void fillNetworkSubstationsGeoData(Network network, List substationsGeoData) { - substationsGeoData.forEach(geoData -> { - Substation foundStation = network.getSubstation(geoData.getId()); + public static void fillNetworkSubstationsGeoData(Network network, Map substationsCoordinates) { + substationsCoordinates.forEach((substationId, coordinate) -> { + Substation foundStation = network.getSubstation(substationId); if (foundStation != null) { foundStation.newExtension(SubstationPositionAdder.class) - .withCoordinate(geoData.getCoordinate()) + .withCoordinate(coordinate) .add(); } }); } - public static void fillNetworkLinesGeoData(Network network, List linesGeoData) { - linesGeoData.forEach(geoData -> { - Line foundLine = network.getLine(geoData.getId()); - if (foundLine != null) { - foundLine.newExtension(LinePositionAdder.class) - .withCoordinates(geoData.getCoordinates()) + public static void fillNetworkLinesGeoData(Network network, Map> linesGeoData) { + AtomicInteger unknownLines = new AtomicInteger(); + linesGeoData.forEach((lineId, lineCoordinates) -> { + Line line = network.getLine(lineId); + if (line != null) { + line.newExtension(LinePositionAdder.class) + .withCoordinates(LineCoordinatesOrdering.order(line, lineCoordinates)) .add(); + } else { + unknownLines.getAndIncrement(); } }); + LOGGER.warn("{} unknown lines discarded", unknownLines.get()); } - } diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/LineGeoDataTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/LineGeoDataTest.java deleted file mode 100644 index a792a88dfc2..00000000000 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/LineGeoDataTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright (c) 2019, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.iidm.geodata.elements; - -import com.powsybl.iidm.network.extensions.Coordinate; -import org.junit.jupiter.api.Test; - -import java.util.Collections; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Chamseddine Benhamed {@literal } - */ -class LineGeoDataTest { - - @Test - void test() { - LineGeoData lineGeoData = new LineGeoData("l", "FR", "FR", "ALAMO", "CORAL", Collections.emptyList()); - - assertEquals("l", lineGeoData.getId()); - assertEquals("FR", lineGeoData.getCountry1()); - assertEquals("FR", lineGeoData.getCountry2()); - assertEquals("ALAMO", lineGeoData.getSubstationStart()); - assertEquals("CORAL", lineGeoData.getSubstationEnd()); - assertTrue(lineGeoData.getCoordinates().isEmpty()); - } -} diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/SubstationGeoDataTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/SubstationGeoDataTest.java deleted file mode 100644 index e8cff1a4fe8..00000000000 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/elements/SubstationGeoDataTest.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright (c) 2019, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.iidm.geodata.elements; - -import com.powsybl.iidm.network.extensions.Coordinate; -import org.junit.jupiter.api.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author Chamseddine Benhamed {@literal } - */ -class SubstationGeoDataTest { - - @Test - void test() { - SubstationGeoData substationGeoData = new SubstationGeoData("id", "FR", new Coordinate(1, 1)); - - assertEquals("id", substationGeoData.getId()); - assertEquals("FR", substationGeoData.getCountry()); - assertEquals(new Coordinate(1, 1), substationGeoData.getCoordinate()); - } -} diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/FileValidatorTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/FileValidatorTest.java index 819833cbed5..888033cf294 100644 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/FileValidatorTest.java +++ b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/FileValidatorTest.java @@ -7,15 +7,18 @@ */ package com.powsybl.iidm.geodata.odre; +import com.powsybl.iidm.geodata.utils.InputUtils; +import org.apache.commons.csv.CSVParser; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author Ahmed Bendaamer {@literal } @@ -23,11 +26,9 @@ */ class FileValidatorTest extends AbstractOdreTest { - @ParameterizedTest + @ParameterizedTest(name = "{0}") @MethodSource("provideTestArguments") - void whenCallingValidate(String descr, String directory, OdreConfig config) throws URISyntaxException { - Path file = Paths.get(getClass() - .getClassLoader().getResource(directory + "substations.csv").toURI()); + void whenCallingValidate(String testName, String directory, OdreConfig config) throws URISyntaxException, IOException { Path aerialLinesFile = Paths.get(getClass() .getClassLoader().getResource(directory + "aerial-lines.csv").toURI()); Path undergroundLinesFile = Paths.get(getClass() @@ -38,19 +39,19 @@ void whenCallingValidate(String descr, String directory, OdreConfig config) thro .getClassLoader().getResource(directory + "substations-error.csv").toURI()); // test substations file validator with valid file - assertTrue(FileValidator.validateSubstations(file, config)); + assertTrue(FileValidator.validateSubstationsHeaders(getCsvParser(substationsFile), config)); // test substations file validator with invalid file - assertFalse(FileValidator.validateSubstations(invalidFile, config)); + assertFalse(FileValidator.validateSubstationsHeaders(getCsvParser(invalidFile), config)); // test lines file validator with valid files - assertEquals(3, FileValidator.validateLines(List.of(substationsFile, aerialLinesFile, undergroundLinesFile), config).size()); - // test lines file validator with 1 invalid file - assertEquals(2, FileValidator.validateLines(List.of(substationsFile, invalidFile, undergroundLinesFile), config).size()); - // test lines file validator with 2 invalid file - assertEquals(1, FileValidator.validateLines(List.of(invalidFile, invalidFile, undergroundLinesFile), config).size()); - // test lines file validator with 3 invalid file - assertEquals(0, FileValidator.validateLines(List.of(invalidFile, invalidFile, invalidFile), config).size()); - // test lines file validator with 4 invalid file - assertEquals(0, FileValidator.validateLines(List.of(invalidFile, invalidFile, invalidFile, invalidFile), config).size()); + assertTrue(FileValidator.validateAerialLinesHeaders(getCsvParser(aerialLinesFile), config)); + assertTrue(FileValidator.validateUndergroundHeaders(getCsvParser(undergroundLinesFile), config)); + // test lines file validator with an invalid file + assertFalse(FileValidator.validateAerialLinesHeaders(getCsvParser(invalidFile), config)); + assertFalse(FileValidator.validateUndergroundHeaders(getCsvParser(invalidFile), config)); + } + + private static CSVParser getCsvParser(Path substationsFile) throws IOException { + return CSVParser.parse(InputUtils.toReader(substationsFile), FileValidator.CSV_FORMAT); } } diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderTest.java index bee5ed12a33..5eca55d8f28 100644 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderTest.java +++ b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataAdderTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; @@ -38,9 +39,9 @@ void setUp() { network = EurostagTutorialExample1Factory.create(); } - @ParameterizedTest + @ParameterizedTest(name = "{0}") @MethodSource("provideTestArguments") - void addSubstationsGeoDataFromFile(String descr, String directory, OdreConfig config) throws URISyntaxException { + void addSubstationsGeoDataFromFile(String descr, String directory, OdreConfig config) throws URISyntaxException, IOException { Path substationsPath = Paths.get(getClass() .getClassLoader().getResource(directory + "substations.csv").toURI()); @@ -59,9 +60,9 @@ void addSubstationsGeoDataFromFile(String descr, String directory, OdreConfig co assertEquals(coord2, position2.getCoordinate()); } - @ParameterizedTest + @ParameterizedTest(name = "{0}") @MethodSource("provideTestArguments") - void addLinesGeoDataFromFile(String descr, String directory, OdreConfig config) throws URISyntaxException { + void addLinesGeoDataFromFile(String descr, String directory, OdreConfig config) throws URISyntaxException, IOException { Path substationsPath = Paths.get(getClass() .getClassLoader().getResource(directory + "substations.csv").toURI()); Path aerialLinesFile = Paths.get(getClass() @@ -69,8 +70,8 @@ void addLinesGeoDataFromFile(String descr, String directory, OdreConfig config) Path undergroundLinesFile = Paths.get(getClass() .getClassLoader().getResource(directory + "underground-lines.csv").toURI()); - OdreGeoDataAdder.fillNetworkLinesGeoDataFromFiles(network, aerialLinesFile, - undergroundLinesFile, substationsPath, config); + OdreGeoDataAdder.fillNetworkSubstationsGeoDataFromFile(network, substationsPath, config); + OdreGeoDataAdder.fillNetworkLinesGeoDataFromFiles(network, aerialLinesFile, undergroundLinesFile, config); Line line = network.getLine("NHV1_NHV2_2"); LinePosition linePosition = line.getExtension(LinePosition.class); diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoaderTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoaderTest.java deleted file mode 100644 index 6768069c3f6..00000000000 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/odre/OdreGeoDataCsvLoaderTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (c) 2024, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.iidm.geodata.odre; - -import com.powsybl.iidm.geodata.elements.LineGeoData; -import org.junit.jupiter.api.Test; - -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author Hugo Kulesza {@literal } - */ -class OdreGeoDataCsvLoaderTest { - - @Test - void validSubstationsLineParsing() throws URISyntaxException { - Path substationsPath = Paths.get(getClass() - .getClassLoader().getResource("valid-line-name/substations.csv").toURI()); - Path undergroundLinesPath = Paths.get(getClass() - .getClassLoader().getResource("valid-line-name/underground-lines.csv").toURI()); - Path linesPath = Paths.get(getClass() - .getClassLoader().getResource("valid-line-name/aerial-lines.csv").toURI()); - - List linesGeodata = OdreGeoDataCsvLoader.getLinesGeoData(linesPath, undergroundLinesPath, substationsPath, AbstractOdreTest.ODRE_CONFIG1); - - assertEquals(2, linesGeodata.size()); - LineGeoData line1Position = linesGeodata.get(0); - assertEquals("POST1L71POST3", line1Position.getId()); - assertEquals("POST1", line1Position.getSubstationEnd()); - assertEquals("", line1Position.getSubstationStart()); - - LineGeoData line2Position = linesGeodata.get(1); - assertEquals("POST1L71POST2", line2Position.getId()); - assertEquals("POST1", line2Position.getSubstationEnd()); - assertEquals("POST2", line2Position.getSubstationStart()); - } - -} diff --git a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdderTest.java b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdderTest.java index a629e66c4b6..a06630bfb98 100644 --- a/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdderTest.java +++ b/iidm/iidm-geodata/src/test/java/com/powsybl/iidm/geodata/utils/NetworkGeoDataExtensionsAdderTest.java @@ -7,8 +7,6 @@ */ package com.powsybl.iidm.geodata.utils; -import com.powsybl.iidm.geodata.elements.LineGeoData; -import com.powsybl.iidm.geodata.elements.SubstationGeoData; import com.powsybl.iidm.network.Line; import com.powsybl.iidm.network.Network; import com.powsybl.iidm.network.Substation; @@ -20,6 +18,7 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import static com.powsybl.iidm.geodata.utils.NetworkGeoDataExtensionsAdder.fillNetworkLinesGeoData; import static com.powsybl.iidm.geodata.utils.NetworkGeoDataExtensionsAdder.fillNetworkSubstationsGeoData; @@ -42,9 +41,7 @@ void setUp() { void addSubstationsPosition() { Coordinate coord1 = new Coordinate(1, 2); Coordinate coord2 = new Coordinate(3, 4); - SubstationGeoData p1GeoData = new SubstationGeoData("P1", "FR", coord1); - SubstationGeoData p2GeoData = new SubstationGeoData("P2", "BE", coord2); - List substationsGeoData = List.of(p1GeoData, p2GeoData); + Map substationsGeoData = Map.of("P1", coord1, "P2", coord2); fillNetworkSubstationsGeoData(network, substationsGeoData); @@ -63,10 +60,9 @@ void addSubstationsPosition() { void addLinesGeoData() { Coordinate coord1 = new Coordinate(1, 2); Coordinate coord2 = new Coordinate(3, 4); - LineGeoData position = new LineGeoData("NHV1_NHV2_2", "FR", "BE", - "P1", "P2", List.of(coord1, coord2)); + var position = Map.of("NHV1_NHV2_2", List.of(coord1, coord2)); - fillNetworkLinesGeoData(network, List.of(position)); + fillNetworkLinesGeoData(network, position); Line line = network.getLine("NHV1_NHV2_2"); LinePosition linePosition = line.getExtension(LinePosition.class); diff --git a/iidm/iidm-geodata/src/test/resources/valid-line-name/aerial-lines.csv b/iidm/iidm-geodata/src/test/resources/valid-line-name/aerial-lines.csv deleted file mode 100644 index 29a14bc6a74..00000000000 --- a/iidm/iidm-geodata/src/test/resources/valid-line-name/aerial-lines.csv +++ /dev/null @@ -1,2 +0,0 @@ -type_ouvrage;code_ligne;nom_ligne;proprietaire_ligne;etat;tension;source;geo_shape;nombre_circuit;identification_2;nom_ouvrage_2;proprietaire_2;identification_3;nom_ouvrage_3;proprietaire_3;identification_4;nom_ouvrage_4;proprietaire_4;identification_5;nom_ouvrage_5;proprietaire_5;geo_point_2d -AERIEN;POST1L71POST2;POST1L71POST2;RTE;EN EXPLOITATION;400kV;RTE;"{""coordinates"": [[1, 1], [2, 2], [3, 3]], ""type"": ""LineString""}";1;;;;;;;;;;;;;2, 2 diff --git a/iidm/iidm-geodata/src/test/resources/valid-line-name/substations.csv b/iidm/iidm-geodata/src/test/resources/valid-line-name/substations.csv deleted file mode 100644 index 7cd060fdf82..00000000000 --- a/iidm/iidm-geodata/src/test/resources/valid-line-name/substations.csv +++ /dev/null @@ -1,3 +0,0 @@ -code_poste;nom_poste;fonction;etat;tension;longitude_poste;latitude_poste;geo_point_poste -POST1;POST1;POSTE DE TRANSFORMATION;E;380kV;3;3;3,3 -POST2;POST2;POSTE DE TRANSFORMATION;E;380kV;1;1;1,1 diff --git a/iidm/iidm-geodata/src/test/resources/valid-line-name/underground-lines.csv b/iidm/iidm-geodata/src/test/resources/valid-line-name/underground-lines.csv deleted file mode 100644 index dbdbb7b8c98..00000000000 --- a/iidm/iidm-geodata/src/test/resources/valid-line-name/underground-lines.csv +++ /dev/null @@ -1,2 +0,0 @@ -type_ouvrage;code_ligne;nom_ligne;proprietaire_ligne;etat;tension;source;geo_shape;nombre_circuit;identification_2;nom_ouvrage_2;proprietaire_2;identification_3;nom_ouvrage_3;proprietaire_3;identification_4;nom_ouvrage_4;proprietaire_4;identification_5;nom_ouvrage_5;proprietaire_5;geo_point_2d -SOUTERRAIN;POST1L71POST3;POST1L71POST3;RTE;EN EXPLOITATION;400kV;RTE;"{""coordinates"": [[1, 1], [2, 2], [3, 3]], ""type"": ""LineString""}";1;;;;;;;;;;;;;2, 2