Merge pull request #556 from afischerdev/jdbc-import

Works for me, but performance test for planet processing still running. Maybe I'll change to prelaod Database Info and matching against a memory map, but I first merge and do eventual changes in a new PR

brouter-map-creator/src/main/java/btools/mapcreator/OsmCutter.java

@@ -12,6 +12,12 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
 import java.util.Map;
 
 import btools.expressions.BExpressionContextNode;
@@ -33,6 +39,19 @@ public class OsmCutter extends MapCreatorBase {
   public RestrictionCutter restrictionCutter;
   public NodeFilter nodeFilter;
 
+
+  Connection conn = null;
+  PreparedStatement psAllTags = null;
+
+  ResultSet rsBrouter = null;
+
+  int cntHighways = 0;
+  int cntWayModified = 0;
+
+  String jdbcurl;
+  Map<String, String> databaseField2Tag;
+  Map<String, Integer> databaseFieldsFound;
+
   public static void main(String[] args) throws Exception {
     System.out.println("*** OsmCutter: cut an osm map in node-tiles + a way file");
     if (args.length != 6 && args.length != 7) {
@@ -55,9 +74,6 @@ public class OsmCutter extends MapCreatorBase {
   private BExpressionContextWay _expctxWay;
   private BExpressionContextNode _expctxNode;
 
-  // private BExpressionContextWay _expctxWayStat;
-  // private BExpressionContextNode _expctxNodeStat;
-
   public void process(File lookupFile, File outTileDir, File wayFile, File relFile, File resFile, File profileFile, File mapFile) throws Exception {
     if (!lookupFile.exists()) {
       throw new IllegalArgumentException("lookup-file: " + lookupFile + " does not exist");
@@ -70,10 +86,6 @@ public class OsmCutter extends MapCreatorBase {
     meta.readMetaData(lookupFile);
     _expctxWay.parseFile(profileFile, "global");
 
-
-    // _expctxWayStat = new BExpressionContextWay( null );
-    // _expctxNodeStat = new BExpressionContextNode( null );
-
     this.outTileDir = outTileDir;
     if (!outTileDir.isDirectory())
       throw new RuntimeException("out tile directory " + outTileDir + " does not exist");
@@ -117,6 +129,9 @@ public class OsmCutter extends MapCreatorBase {
     return "records read: " + recordCnt + " nodes=" + nodesParsed + " ways=" + waysParsed + " rels=" + relsParsed + " changesets=" + changesetsParsed;
   }
 
+  public void setJdbcUrl(String url) {
+    this.jdbcurl = url;
+  }
 
   @Override
   public void nextNode(NodeData n) throws Exception {
@@ -164,6 +179,78 @@ public class OsmCutter extends MapCreatorBase {
     }
   }
 
+  private void generateTagsFromDatabase(long osm_id, Map<String, String> map) {
+
+    if (jdbcurl == null) return;
+
+    try {
+      // is the database allready connected?
+      if (conn == null) {
+
+        String sql_all_tags = "SELECT *  from all_tags where losmid = ?";
+
+        System.out.println("OsmCutter start connection to the database........" + jdbcurl);
+
+        conn = DriverManager.getConnection(jdbcurl);
+        psAllTags = conn.prepareStatement(sql_all_tags);
+
+        databaseField2Tag = new HashMap<>();
+        databaseField2Tag.put("noise_class", "estimated_noise_class");
+        databaseField2Tag.put("river_class", "estimated_river_class");
+        databaseField2Tag.put("forest_class", "estimated_forest_class");
+        databaseField2Tag.put("town_class", "estimated_town_class");
+        databaseField2Tag.put("traffic_class", "estimated_traffic_class");
+
+        databaseFieldsFound = new HashMap<>();
+        for (String key : databaseField2Tag.keySet()) {
+          databaseFieldsFound.put(key, 0);
+        }
+
+
+        System.out.println("OsmCutter connect to the database ok........");
+
+      }
+
+      for (Map.Entry<String, String> e : map.entrySet()) {
+        if (e.getKey().equals("highway")) {
+          cntHighways = cntHighways + 1;
+
+          psAllTags.setLong(1, osm_id);
+
+          // process the results
+          rsBrouter = psAllTags.executeQuery();
+
+          if (rsBrouter.next()) {
+
+            cntWayModified = cntWayModified + 1;
+            for (String key : databaseField2Tag.keySet()) {
+              if (rsBrouter.getString(key) != null) {
+                map.put(databaseField2Tag.get(key), rsBrouter.getString(key));
+                databaseFieldsFound.put(key, databaseFieldsFound.get(key) + 1);
+              }
+            }
+          }
+          if ((cntHighways % 100000) == 0) {
+            String out = "HW processed=" + cntHighways + " HW modifs=" + cntWayModified;
+            for (String key : databaseFieldsFound.keySet()) {
+              out += " " + key + "=" + databaseFieldsFound.get(key);
+            }
+            System.out.println(out);
+          }
+
+          return;
+        }
+      }
+
+    } catch (SQLException g) {
+      System.err.format(" OsmCutter execute sql .. SQL State: %s\n%s\n", g.getSQLState(), g.getMessage());
+      System.exit(1);
+    } catch (Exception f) {
+      f.printStackTrace();
+      System.exit(1);
+    }
+  }
+
 
   @Override
   public void nextWay(WayData w) throws Exception {
@@ -173,13 +260,14 @@ public class OsmCutter extends MapCreatorBase {
     // encode tags
     if (w.getTagsOrNull() == null) return;
 
+    generateTagsFromDatabase(w.wid, w.getTagsOrNull());
+
     generatePseudoTags(w.getTagsOrNull());
 
     int[] lookupData = _expctxWay.createNewLookupData();
     for (String key : w.getTagsOrNull().keySet()) {
       String value = w.getTag(key);
       _expctxWay.addLookupValue(key, value.replace(' ', '_'), lookupData);
-      // _expctxWayStat.addLookupValue( key, value, null );
     }
     w.description = _expctxWay.encode(lookupData);
 

brouter-map-creator/src/main/java/btools/mapcreator/OsmFastCutter.java

@@ -12,8 +12,8 @@ import java.io.File;
 public class OsmFastCutter extends MapCreatorBase {
   public static void main(String[] args) throws Exception {
     System.out.println("*** OsmFastCutter: cut an osm map in node-tiles + way-tiles");
-    if (args.length != 11 && args.length != 12) {
-      String common = "java OsmFastCutter <lookup-file> <node-dir> <way-dir> <node55-dir> <way55-dir> <border-file> <out-rel-file> <out-res-file> <filter-profile> <report-profile> <check-profile>";
+    if (args.length != 11 && args.length != 12 && args.length != 13) {
+      String common = "java OsmFastCutter <lookup-file> <node-dir> <way-dir> <node55-dir> <way55-dir> <border-file> <out-rel-file> <out-res-file> <filter-profile> <report-profile> <check-profile> <map-file> [jdbc-url]";
 
       System.out.println("usage: bzip2 -dc <map> | " + common);
       System.out.println("or   : " + common + " <inputfile> ");
@@ -33,12 +33,14 @@ public class OsmFastCutter extends MapCreatorBase {
       , new File(args[9])
       , new File(args[10])
       , args.length > 11 ? new File(args[11]) : null
+      , args.length > 12 ? args[12] : null
     );
   }
 
-  public static void doCut(File lookupFile, File nodeDir, File wayDir, File node55Dir, File way55Dir, File borderFile, File relFile, File resFile, File profileAll, File profileReport, File profileCheck, File mapFile) throws Exception {
+  public static void doCut(File lookupFile, File nodeDir, File wayDir, File node55Dir, File way55Dir, File borderFile, File relFile, File resFile, File profileAll, File profileReport, File profileCheck, File mapFile, String jdbcurl) throws Exception {
     // **** run OsmCutter ****
     OsmCutter cutter = new OsmCutter();
+    if (jdbcurl != null) cutter.setJdbcUrl(jdbcurl);
 
     // ... inject WayCutter
     cutter.wayCutter = new WayCutter();

brouter-map-creator/src/test/java/btools/mapcreator/MapcreatorTest.java

@@ -35,7 +35,7 @@ public class MapcreatorTest {
     File profileCheck = new File(profileDir, "softaccess.brf");
     File borderFile = new File(tmpdir, "bordernids.dat");
 
-    new OsmFastCutter().doCut(lookupFile, nodes, ways, nodes55, ways55, borderFile, relFile, resFile, profileAll, profileReport, profileCheck, mapFile);
+    new OsmFastCutter().doCut(lookupFile, nodes, ways, nodes55, ways55, borderFile, relFile, resFile, profileAll, profileReport, profileCheck, mapFile, null);
 
 
     // run PosUnifier