added crc logic for datafile access

brouter-mapaccess/pom.xml

@@ -10,4 +10,12 @@
     </parent>
     <artifactId>brouter-mapaccess</artifactId>
     <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.btools</groupId>
+            <artifactId>brouter-util</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
 </project>

brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java

@@ -8,6 +8,8 @@ package btools.mapaccess;
 import java.util.*;
 import java.io.*;
 
+import btools.util.Crc32;
+
 final class MicroCache
 {
   private long[] faid;
@@ -32,10 +34,10 @@ final class MicroCache
 
      int subIdx = (latIdx80-80*latDegree)*80 + (lonIdx80-80*lonDegree);
 
-     try
      {
        ab = iobuffer;
        int asize = segfile.getDataInputForSubIdx(subIdx, ab);
+
        if ( asize == 0 )
        {
          return;
@@ -48,8 +50,30 @@ final class MicroCache
        aboffset = 0;
        size = readInt();
 
+       // get net size
+       int nbytes = 0;
+       for(int i = 0; i<size; i++)
+       {
+         int ilon = readShort();
+         int ilat = readShort();
+         int bodySize = readInt();
+         if ( ilon == Short.MAX_VALUE )        
+         {
+           int crc = Crc32.crc( ab, 0, aboffset-8 );
+           if ( crc != readInt() )
+           {
+             throw new IOException( "checkum error" );
+           }
+           size = i;
+           break;
+         }
+         aboffset += bodySize;
+         nbytes += bodySize;
+       }
+
        // new array with only net data
-       byte[] nab = new byte[asize - 4 - size*8];
+       byte[] nab = new byte[nbytes];
+       aboffset = 4;
        int noffset = 0;
        faid = new long[size];
        fapos = new int[size];
@@ -71,11 +95,9 @@ final class MicroCache
          aboffset += bodySize;
          noffset += bodySize;
        }
+
        ab = nab;
      }
-     catch( EOFException eof )
-     {
-     }
   }
 
   public int getSize()

brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java

@@ -16,8 +16,7 @@ public final class NodesCache
   private boolean carMode;
   private String currentFileName;
 
-  private HashMap<String,RandomAccessFile> fileCache;
-  private HashMap<String,long[]> indexCache;
+  private HashMap<String,PhysicalFile> fileCache;
   private byte[] iobuffer;
   
   private OsmFile[][] fileRows = new OsmFile[180][];
@@ -37,14 +36,12 @@ public final class NodesCache
     if ( oldCache != null )
     {
       fileCache = oldCache.fileCache;
-      indexCache = oldCache.indexCache;
       iobuffer = oldCache.iobuffer;
       oom_carsubset_hint = oldCache.oom_carsubset_hint;
     }
     else
     {
-      fileCache = new HashMap<String,RandomAccessFile>(4);
-      indexCache = new HashMap<String,long[]>(4);
+      fileCache = new HashMap<String,PhysicalFile>(4);
       iobuffer = new byte[65636];
     }
   }
@@ -144,6 +141,7 @@ public final class NodesCache
 
     currentFileName = filenameBase + ".rd5/cd5";
 
+    PhysicalFile ra = null;
     if ( !fileCache.containsKey( filenameBase ) )
     {
       File f = null;
@@ -158,39 +156,17 @@ public final class NodesCache
         if ( fullFile.exists() ) f = fullFile;
         if ( carMode && f != null ) oom_carsubset_hint = true;
       }
-      RandomAccessFile ra = f != null ? new RandomAccessFile( f, "r" ) : null;
-      fileCache.put( filenameBase, ra );
-      if ( ra != null )
+      if ( f != null )
       {
-        long[] fileIndex = new long[25];
-        ra.readFully( iobuffer, 0, 200 );
-        ByteDataReader dis = new ByteDataReader( iobuffer );
-        for( int i=0; i<25; i++ )
-        {
-          long lv = dis.readLong();
-          short readVersion = (short)(lv >> 48);
-          if ( readVersion != lookupVersion )
-          {
-            throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
-                     + lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
-          }
-          fileIndex[i] = lv & 0xffffffffffffL;
-        }
-        indexCache.put( filenameBase, fileIndex );
+        currentFileName = f.getName();
+        ra = new PhysicalFile( f, iobuffer, lookupVersion );
       }
+      fileCache.put( filenameBase, ra );
     }
-    RandomAccessFile ra = fileCache.get( filenameBase );
-    long startPos = 0L;
-    if ( ra != null )
-    {
-      long[] index = indexCache.get( filenameBase );
-      startPos = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
-      if ( startPos == index[ tileIndex] ) ra = null;
-    }
-    OsmFile osmf = new OsmFile( ra, startPos, iobuffer );
+    ra = fileCache.get( filenameBase );
+    OsmFile osmf = new OsmFile( ra, tileIndex, iobuffer );
     osmf.lonDegree = lonDegree;
     osmf.latDegree = latDegree;
-    osmf.filename = currentFileName;
     return osmf;
   }
 
@@ -208,11 +184,11 @@ public final class NodesCache
 
   public void close()
   {
-    for( RandomAccessFile f: fileCache.values() )
+    for( PhysicalFile f: fileCache.values() )
     {
       try
       {
-        f.close();
+        f.ra.close();
       }
       catch( IOException ioe )
       {

brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java

@@ -7,6 +7,7 @@ package btools.mapaccess;
 
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import btools.util.Crc32;
 
 final class OsmFile
 {
@@ -21,16 +22,31 @@ final class OsmFile
 
   public String filename;
 
-  public OsmFile( RandomAccessFile rafile, long startPos, byte[] iobuffer ) throws Exception
+  public OsmFile( PhysicalFile rafile, int tileIndex, byte[] iobuffer ) throws Exception
   {
-    fileOffset = startPos;
     if ( rafile != null )
     {
-      is = rafile;
+      filename = rafile.fileName;
+
+      long[] index = rafile.fileIndex;
+      fileOffset = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
+      if ( fileOffset == index[ tileIndex] ) return; // empty
+    	
+      is = rafile.ra;
       posIdx = new int[6400];
       microCaches = new MicroCache[6400];
       is.seek( fileOffset );
       is.readFully( iobuffer, 0, 25600 );
+      
+      if ( rafile.fileHeaderCrcs != null )
+      {
+        int headerCrc = Crc32.crc( iobuffer, 0, 25600 );
+        if ( rafile.fileHeaderCrcs[tileIndex] != headerCrc )
+        {
+          throw new IOException( "sub index checksum error" );
+        }
+      }
+      
       ByteDataReader dis = new ByteDataReader( iobuffer );
       for( int i=0; i<6400; i++ )
       {
@@ -54,7 +70,7 @@ final class OsmFile
        is.seek( fileOffset + startPos );
        if ( size <= iobuffer.length )
        {
-         is.readFully( iobuffer );
+         is.readFully( iobuffer, 0, size );
        }
      }
      return size;

brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java

@@ -0,0 +1,69 @@
+/**
+ * cache for a single square
+ *
+ * @author ab
+ */
+package btools.mapaccess;
+
+import java.io.*;
+import btools.util.Crc32;
+
+final class PhysicalFile
+{
+  RandomAccessFile ra = null;
+  long[] fileIndex = new long[25];
+  int[] fileHeaderCrcs;
+   
+  private int fileIndexCrc;
+  public long creationTime;
+
+  String fileName;
+  
+  public PhysicalFile( File f, byte[] iobuffer, int lookupVersion ) throws Exception
+  {
+    fileName = f.getName();
+
+    ra = new RandomAccessFile( f, "r" );
+    ra.readFully( iobuffer, 0, 200 );
+    fileIndexCrc = Crc32.crc( iobuffer, 0, 200 );
+    ByteDataReader dis = new ByteDataReader( iobuffer );
+    for( int i=0; i<25; i++ )
+    {
+      long lv = dis.readLong();
+      short readVersion = (short)(lv >> 48);
+      if ( readVersion != lookupVersion )
+      {
+        throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
+                 + lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
+      }
+      fileIndex[i] = lv & 0xffffffffffffL;
+    }
+
+    // read some extra info from the end of the file, if present
+    long len = ra.length();
+
+    long pos = fileIndex[24];
+    int extraLen = 8 + 26*4;
+
+    if ( len == pos ) return; // old format o.k.
+
+    if ( len < pos+extraLen ) // > is o.k. for future extensions!
+    {
+      throw new IOException( "file of size " + len + " + too short, should be " + (pos+extraLen) );
+    }
+    
+    ra.seek( pos );
+    ra.readFully( iobuffer, 0, extraLen );
+    dis = new ByteDataReader( iobuffer );
+    creationTime = dis.readLong();
+    if ( dis.readInt() != fileIndexCrc )
+    {
+      throw new IOException( "top index checksum error" );
+    }
+    fileHeaderCrcs = new int[25];
+    for( int i=0; i<25; i++ )
+    {
+      fileHeaderCrcs[i] = dis.readInt();
+    }
+  }
+}