Skip to content

Commit

Permalink
Merge pull request #27 from ibi-group/graph-builder-improvements
Browse files Browse the repository at this point in the history
Graph builder improvements
  • Loading branch information
landonreed authored Feb 25, 2020
2 parents c0e761e + 4838f3f commit 08c508a
Show file tree
Hide file tree
Showing 43 changed files with 353 additions and 1,215 deletions.
113 changes: 106 additions & 7 deletions src/main/java/org/opentripplanner/graph_builder/AnnotationsToHTML.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

import org.apache.commons.io.FileUtils;
import org.opentripplanner.graph_builder.annotation.GraphBuilderAnnotation;
import org.opentripplanner.graph_builder.module.GraphBuilderModuleSummary;
import org.opentripplanner.graph_builder.module.GraphBuilderTaskSummary;
import org.opentripplanner.graph_builder.services.GraphBuilderModule;
import org.opentripplanner.routing.graph.Graph;
import org.slf4j.Logger;
Expand Down Expand Up @@ -59,7 +61,7 @@ public AnnotationsToHTML (File outpath, int maxNumberOfAnnotationsPerFile) {


@Override
public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
public void buildGraph(Graph graph, GraphBuilderModuleSummary graphBuilderModuleSummary) {

if (outPath == null) {
LOG.error("Saving folder is empty!");
Expand Down Expand Up @@ -118,7 +120,7 @@ public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
}

try {
HTMLWriter indexFileWriter = new HTMLWriter("index", (Multimap<String, String>)null);
HTMLWriter indexFileWriter = new HTMLWriter("index", graph.getBuilderSummaries());
indexFileWriter.writeFile(annotationClassOccurences, true);
} catch (FileNotFoundException e) {
LOG.error("Index file coudn't be created:{}", e);
Expand Down Expand Up @@ -178,7 +180,6 @@ public void checkInputs() {
private void addAnnotation(GraphBuilderAnnotation annotation) {
String className = annotation.getClass().getSimpleName();
annotations.put(className, annotation.getHTMLMessage());

}

class HTMLWriter {
Expand All @@ -188,6 +189,8 @@ class HTMLWriter {

private String annotationClassName;

private List<GraphBuilderModuleSummary> graphBuilderModuleSummaries = null;

public HTMLWriter(String key, Collection<String> annotations) throws FileNotFoundException {
LOG.debug("Making file: {}", key);
File newFile = new File(outPath, key +".html");
Expand All @@ -198,14 +201,14 @@ public HTMLWriter(String key, Collection<String> annotations) throws FileNotFoun
annotationClassName = key;
}

public HTMLWriter(String filename, Multimap<String, String> curMap)
public HTMLWriter(String filename, List<GraphBuilderModuleSummary> graphBuilderModuleSummaries)
throws FileNotFoundException {
LOG.debug("Making file: {}", filename);
File newFile = new File(outPath, filename +".html");
FileOutputStream fileOutputStream = new FileOutputStream(newFile);
this.out = new PrintStream(fileOutputStream);
writerAnnotations = curMap;
annotationClassName = filename;
this.graphBuilderModuleSummaries = graphBuilderModuleSummaries;
}

private void writeFile(Multiset<String> classes, boolean isIndexFile) {
Expand All @@ -217,6 +220,10 @@ private void writeFile(Multiset<String> classes, boolean isIndexFile) {
println(
"<link rel='stylesheet' href='http://yui.yahooapis.com/pure/0.5.0/pure-min.css'>");
String css = "\t\t<style>\n"
+ "\n"
+ "\t\t\tbody {\n"
+ "\t\t\t\tmargin:8px;\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t\tbutton.pure-button {\n"
+ "\t\t\t\tmargin:5px;\n"
Expand Down Expand Up @@ -250,13 +257,38 @@ private void writeFile(Multiset<String> classes, boolean isIndexFile) {
+ "\t\t\t\tbackground: rgb(66, 184, 221); /* this is a light blue */\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t\tth, td {\n"
+ "\t\t\t\tborder: 1px solid black;\n"
+ "\t\t\t\tpadding: 5px;\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t\ttable {\n"
+ "\t\t\t\tmargin: 5px;\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t\t.subtasks table {\n"
+ "\t\t\t\tmargin: 0;\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t\t.subtasks td {\n"
+ "\t\t\t\tborder: none;\n"
+ "\t\t\t\tmin-width: 145px;\n"
+ "\t\t\t}\n"
+ "\n"
+ "\t\t</style>\n"
+ "";
println(css);
println("</head><body>");
println(String.format("<h1>OpenTripPlanner annotations log for %s</h1>", annotationClassName));
println(isIndexFile
? "<h1>OpenTripPlanner Graph Build Report</h1>"
: String.format("<h1>OpenTripPlanner annotations log for %s</h1>", annotationClassName));
println("<h2>Graph report for " + outPath.getParentFile() + "Graph.obj</h2>");
println("<p>");
// add link for home page
println(isIndexFile
? "<button class='pure-button pure-button-disabled'>Home</button>"
: "<a class='pure-button' href=\"index.html\">Home</a>"
);
//adds links to the other HTML files
for (Multiset.Entry<String> htmlAnnotationClass : classes.entrySet()) {
String label_name = htmlAnnotationClass.getElement();
Expand All @@ -276,7 +308,9 @@ private void writeFile(Multiset<String> classes, boolean isIndexFile) {
}
}
println("</p>");
if (!isIndexFile) {
if (isIndexFile) {
writeIndexContent();
} else {
println("<ul id=\"log\">");
writeAnnotations();
println("</ul>");
Expand All @@ -287,6 +321,71 @@ private void writeFile(Multiset<String> classes, boolean isIndexFile) {
close();
}

/**
* Writes the index page contents.
*/
private void writeIndexContent() {
println("<h3>Graph builder module report</h3>");
println("<table>");
println("<thead>");
println("<tr><th>Graph Builder Module</th><th>Processing time</th><th>Subtasks</th></tr>");
println("</thead>");
println("<tbody>");
long totalDuration = 0;
for (GraphBuilderModuleSummary summary : graphBuilderModuleSummaries) {
totalDuration += summary.getDuration();
}

for (GraphBuilderModuleSummary summary: graphBuilderModuleSummaries) {
StringBuilder summaryLine = new StringBuilder();
summaryLine.append(
String.format(
"<tr><td>%s</td><td>%.1fs (%.0f%%)</td><td class=\"subtasks\">",
summary.getName(),
summary.getDuration() / 1000.0,
summary.getDuration() * 1.0 / totalDuration * 100
)
);
if (summary.getSubTasks().size() > 0) {
summaryLine.append("<table>");
for (GraphBuilderTaskSummary subTask : summary.getSubTasks()) {
String subTaskName = subTask.getName();
summaryLine.append(
String.format(
"<tr><td>%s</td><td>%.1fs (%.0f%%)</td></tr>",
subTaskName,
subTask.getDuration() / 1000.0,
subTask.getDuration() * 1.0 / summary.getDuration() * 100
)
);
}
summaryLine.append("</table>");
}
summaryLine.append("</td></tr>");
println(summaryLine.toString());
}
println("</tbody>");
println("</table>");

println("<h3>Annotations</h3>");
println("<table>");
println("<thead>");
println("<tr><th>Annotation Class</th><th>Number of annotations</th></tr>");
println("</thead>");
println("<tbody>");
for (String className : annotations.keySet()) {
println(
String.format(
"<tr><td>%s</td><td>%d</td></tr>",
className,
annotations.get(className).size()
)
);
}
println("</tbody>");
println("</table>");
}

/**
* Writes annotations as LI html elements
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
import org.opentripplanner.graph_builder.model.GtfsBundle;
import org.opentripplanner.graph_builder.module.DirectTransferGenerator;
import org.opentripplanner.graph_builder.module.EmbedConfig;
import org.opentripplanner.graph_builder.module.GraphBuilderModuleSummary;
import org.opentripplanner.graph_builder.module.GtfsModule;
import org.opentripplanner.graph_builder.module.PruneFloatingIslands;
import org.opentripplanner.graph_builder.module.StreetLinkerModule;
import org.opentripplanner.graph_builder.module.TransitToTaggedStopsModule;
import org.opentripplanner.graph_builder.module.map.BusRouteStreetMatcher;
import org.opentripplanner.graph_builder.module.ned.DegreeGridNEDTileSource;
import org.opentripplanner.graph_builder.module.ned.ElevationModule;
import org.opentripplanner.graph_builder.module.ned.GeotiffGridCoverageFactoryImpl;
Expand Down Expand Up @@ -135,10 +135,14 @@ public void run() {
for (GraphBuilderModule builder : _graphBuilderModules) {
builder.checkInputs();
}

HashMap<Class<?>, Object> extra = new HashMap<Class<?>, Object>();
for (GraphBuilderModule load : _graphBuilderModules)
load.buildGraph(graph, extra);

for (GraphBuilderModule module : _graphBuilderModules) {
GraphBuilderModuleSummary moduleSummary = new GraphBuilderModuleSummary(module);
LOG.info(moduleSummary.start());
module.buildGraph(graph, moduleSummary);
LOG.info(moduleSummary.finish());
graph.addGraphBuilderSummary(moduleSummary);
}

graph.summarizeBuilderAnnotations();
if (serializeGraph) {
Expand Down Expand Up @@ -258,9 +262,6 @@ public static GraphBuilder forDirectory(CommandLineParameters params, File dir)
gtfsModule.setFareServiceFactory(builderParams.fareServiceFactory);
graphBuilder.addModule(gtfsModule);
if ( hasOSM ) {
if (builderParams.matchBusRoutesToStreets) {
graphBuilder.addModule(new BusRouteStreetMatcher());
}
graphBuilder.addModule(new TransitToTaggedStopsModule());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;

import org.opentripplanner.common.geometry.SphericalDistanceLibrary;
Expand Down Expand Up @@ -42,7 +41,7 @@ public List<String> getPrerequisites() {
private static final double MAX_VERTEX_SHAPE_ERROR = 150;

@Override
public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
public void buildGraph(Graph graph, GraphBuilderModuleSummary graphBuilderModuleSummary) {
for (Vertex gv : graph.getVertices()) {
if (Double.isNaN(gv.getCoordinate().x) || Double.isNaN(gv.getCoordinate().y)) {
LOG.warn("Vertex " + gv + " has NaN location; this will cause doom.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@
import org.opentripplanner.routing.edgetype.SimpleTransfer;
import org.opentripplanner.routing.graph.Edge;
import org.opentripplanner.routing.graph.Graph;
import org.opentripplanner.routing.graph.GraphIndex;
import org.opentripplanner.routing.vertextype.TransitStop;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
Expand Down Expand Up @@ -46,7 +44,7 @@ public DirectTransferGenerator (double radiusMeters) {
}

@Override
public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
public void buildGraph(Graph graph, GraphBuilderModuleSummary graphBuilderModuleSummary) {
// Make sure the graph index has been initialized because it is needed by the nearby stop finder. Don't
// recalculate the street index because it is only used for finding nearby transit stops which should already
// have been indexed properly in a previous build module.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
import org.opentripplanner.graph_builder.services.GraphBuilderModule;
import org.opentripplanner.routing.graph.Graph;

import java.util.HashMap;

/**
* A graph builder that will embed the JSON graph builder and router configuration into the Graph.
* This allows us to check whether a graph was built with a different configuration file than one found on disk,
Expand All @@ -27,7 +25,7 @@ public EmbedConfig(JsonNode builderConfig, JsonNode routerConfig) {
}

@Override
public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
public void buildGraph(Graph graph, GraphBuilderModuleSummary graphBuilderModuleSummary) {
try {
graph.builderConfig = serializedConfiguration(builderConfig);
graph.routerConfig = serializedConfiguration(routerConfig);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package org.opentripplanner.graph_builder.module;

import org.opentripplanner.graph_builder.services.GraphBuilderModule;

import java.util.ArrayList;
import java.util.List;

/**
* Used to store statistics about various how various graph build modules performed.
*/
public class GraphBuilderModuleSummary extends GraphBuilderSummary {
private List<GraphBuilderTaskSummary> subTasks = new ArrayList<>();

public GraphBuilderModuleSummary(GraphBuilderModule module) {
this.name = module.getClass().getSimpleName();
}

@Override
public String getLogDisplayName() {
return String.format("graph builder module: %s", name);
}

public GraphBuilderTaskSummary addSubTask(String name) {
GraphBuilderTaskSummary subTask = new GraphBuilderTaskSummary(this, name);
subTasks.add(subTask);
return subTask;
}

public List<GraphBuilderTaskSummary> getSubTasks() {
return subTasks;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package org.opentripplanner.graph_builder.module;

public abstract class GraphBuilderSummary {
private long endTime;
protected String name;
private long startTime = 0;

public long getDuration() {
return endTime - startTime;
}

public String getName() { return name; }

public abstract String getLogDisplayName();

public String finish() {
if (this.startTime == 0) {
throw new IllegalStateException("GraphBuilderSummary instance finished before it was started!");
}
this.endTime = System.currentTimeMillis();
return String.format(
"Finished %s in %.1f seconds",
getLogDisplayName(),
getDuration() / 1000.0
);
}

public String start() {
if (this.startTime != 0) {
throw new IllegalStateException("GraphBuilderSummary instance has already been started!");
}
this.startTime = System.currentTimeMillis();
return String.format("Starting %s", getLogDisplayName());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package org.opentripplanner.graph_builder.module;

public class GraphBuilderTaskSummary extends GraphBuilderSummary {
private final GraphBuilderModuleSummary parentModuleSummary;

public GraphBuilderTaskSummary(GraphBuilderModuleSummary graphBuilderModuleSummary, String name) {
this.name = name;
this.parentModuleSummary = graphBuilderModuleSummary;
}

@Override
public String getLogDisplayName() {
return String.format("%s subtask: %s", parentModuleSummary.getLogDisplayName(), name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;

import org.opentripplanner.graph_builder.services.GraphBuilderModule;
Expand Down Expand Up @@ -32,7 +31,7 @@ public List<String> getPrerequisites() {
private static final Logger LOG = LoggerFactory.getLogger(GraphCoherencyCheckerModule.class);

@Override
public void buildGraph(Graph graph, HashMap<Class<?>, Object> extra) {
public void buildGraph(Graph graph, GraphBuilderModuleSummary graphBuilderModuleSummary) {
boolean coherent = true;
LOG.info("checking graph coherency...");
for (Vertex v : graph.getVertices()) {
Expand Down
Loading

0 comments on commit 08c508a

Please sign in to comment.