ben/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

added crc logic for datafile access

Browse source
This commit is contained in:
Arndt Brenschede 2014-02-02 17:04:17 +01:00
parent e88465f5ec
commit b4aa961f2d
26 changed files with 621 additions and 1364 deletions
Download patch file Download diff file Expand all files Collapse all files

32
brouter-mapaccess/src/main/java/btools/mapaccess/MicroCache.java
View file

@ -8,6 +8,8 @@ package btools.mapaccess;
import java.util.*;
import java.io.*;
import btools.util.Crc32;
final class MicroCache
{
private long[] faid;
@ -32,10 +34,10 @@ final class MicroCache
int subIdx = (latIdx80-80*latDegree)*80 + (lonIdx80-80*lonDegree);
try
{
ab = iobuffer;
int asize = segfile.getDataInputForSubIdx(subIdx, ab);
if ( asize == 0 )
{
return;
@ -48,8 +50,30 @@ final class MicroCache
aboffset = 0;
size = readInt();
// get net size
int nbytes = 0;
for(int i = 0; i<size; i++)
{
int ilon = readShort();
int ilat = readShort();
int bodySize = readInt();
if ( ilon == Short.MAX_VALUE )
{
int crc = Crc32.crc( ab, 0, aboffset-8 );
if ( crc != readInt() )
{
throw new IOException( "checkum error" );
}
size = i;
break;
}
aboffset += bodySize;
nbytes += bodySize;
}
// new array with only net data
byte[] nab = new byte[asize - 4 - size*8];
byte[] nab = new byte[nbytes];
aboffset = 4;
int noffset = 0;
faid = new long[size];
fapos = new int[size];
@ -71,11 +95,9 @@ final class MicroCache
aboffset += bodySize;
noffset += bodySize;
}
ab = nab;
}
catch( EOFException eof )
{
}
}
public int getSize()

46
brouter-mapaccess/src/main/java/btools/mapaccess/NodesCache.java
View file

@ -16,8 +16,7 @@ public final class NodesCache
private boolean carMode;
private String currentFileName;
private HashMap<String,RandomAccessFile> fileCache;
private HashMap<String,long[]> indexCache;
private HashMap<String,PhysicalFile> fileCache;
private byte[] iobuffer;
private OsmFile[][] fileRows = new OsmFile[180][];
@ -37,14 +36,12 @@ public final class NodesCache
if ( oldCache != null )
{
fileCache = oldCache.fileCache;
indexCache = oldCache.indexCache;
iobuffer = oldCache.iobuffer;
oom_carsubset_hint = oldCache.oom_carsubset_hint;
}
else
{
fileCache = new HashMap<String,RandomAccessFile>(4);
indexCache = new HashMap<String,long[]>(4);
fileCache = new HashMap<String,PhysicalFile>(4);
iobuffer = new byte[65636];
}
}
@ -144,6 +141,7 @@ public final class NodesCache
currentFileName = filenameBase + ".rd5/cd5";
PhysicalFile ra = null;
if ( !fileCache.containsKey( filenameBase ) )
{
File f = null;
@ -158,39 +156,17 @@ public final class NodesCache
if ( fullFile.exists() ) f = fullFile;
if ( carMode && f != null ) oom_carsubset_hint = true;
}
RandomAccessFile ra = f != null ? new RandomAccessFile( f, "r" ) : null;
fileCache.put( filenameBase, ra );
if ( ra != null )
if ( f != null )
{
long[] fileIndex = new long[25];
ra.readFully( iobuffer, 0, 200 );
ByteDataReader dis = new ByteDataReader( iobuffer );
for( int i=0; i<25; i++ )
{
long lv = dis.readLong();
short readVersion = (short)(lv >> 48);
if ( readVersion != lookupVersion )
{
throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
+ lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
}
fileIndex[i] = lv & 0xffffffffffffL;
}
indexCache.put( filenameBase, fileIndex );
currentFileName = f.getName();
ra = new PhysicalFile( f, iobuffer, lookupVersion );
}
fileCache.put( filenameBase, ra );
}
RandomAccessFile ra = fileCache.get( filenameBase );
long startPos = 0L;
if ( ra != null )
{
long[] index = indexCache.get( filenameBase );
startPos = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
if ( startPos == index[ tileIndex] ) ra = null;
}
OsmFile osmf = new OsmFile( ra, startPos, iobuffer );
ra = fileCache.get( filenameBase );
OsmFile osmf = new OsmFile( ra, tileIndex, iobuffer );
osmf.lonDegree = lonDegree;
osmf.latDegree = latDegree;
osmf.filename = currentFileName;
return osmf;
}
@ -208,11 +184,11 @@ public final class NodesCache
public void close()
{
for( RandomAccessFile f: fileCache.values() )
for( PhysicalFile f: fileCache.values() )
{
try
{
f.close();
f.ra.close();
}
catch( IOException ioe )
{

24
brouter-mapaccess/src/main/java/btools/mapaccess/OsmFile.java
View file

@ -7,6 +7,7 @@ package btools.mapaccess;
import java.io.IOException;
import java.io.RandomAccessFile;
import btools.util.Crc32;
final class OsmFile
{
@ -21,16 +22,31 @@ final class OsmFile
public String filename;
public OsmFile( RandomAccessFile rafile, long startPos, byte[] iobuffer ) throws Exception
public OsmFile( PhysicalFile rafile, int tileIndex, byte[] iobuffer ) throws Exception
{
fileOffset = startPos;
if ( rafile != null )
{
is = rafile;
filename = rafile.fileName;
long[] index = rafile.fileIndex;
fileOffset = tileIndex > 0 ? index[ tileIndex-1 ] : 200L;
if ( fileOffset == index[ tileIndex] ) return; // empty
is = rafile.ra;
posIdx = new int[6400];
microCaches = new MicroCache[6400];
is.seek( fileOffset );
is.readFully( iobuffer, 0, 25600 );
if ( rafile.fileHeaderCrcs != null )
{
int headerCrc = Crc32.crc( iobuffer, 0, 25600 );
if ( rafile.fileHeaderCrcs[tileIndex] != headerCrc )
{
throw new IOException( "sub index checksum error" );
}
}
ByteDataReader dis = new ByteDataReader( iobuffer );
for( int i=0; i<6400; i++ )
{
@ -54,7 +70,7 @@ final class OsmFile
is.seek( fileOffset + startPos );
if ( size <= iobuffer.length )
{
is.readFully( iobuffer );
is.readFully( iobuffer, 0, size );
}
}
return size;

69
brouter-mapaccess/src/main/java/btools/mapaccess/PhysicalFile.java Normal file
View file

@ -0,0 +1,69 @@
/**
* cache for a single square
*
* @author ab
*/
package btools.mapaccess;
import java.io.*;
import btools.util.Crc32;
final class PhysicalFile
{
RandomAccessFile ra = null;
long[] fileIndex = new long[25];
int[] fileHeaderCrcs;
private int fileIndexCrc;
public long creationTime;
String fileName;
public PhysicalFile( File f, byte[] iobuffer, int lookupVersion ) throws Exception
{
fileName = f.getName();
ra = new RandomAccessFile( f, "r" );
ra.readFully( iobuffer, 0, 200 );
fileIndexCrc = Crc32.crc( iobuffer, 0, 200 );
ByteDataReader dis = new ByteDataReader( iobuffer );
for( int i=0; i<25; i++ )
{
long lv = dis.readLong();
short readVersion = (short)(lv >> 48);
if ( readVersion != lookupVersion )
{
throw new IllegalArgumentException( "lookup version mismatch (old rd5?) lookups.dat="
+ lookupVersion + " " + f. getAbsolutePath() + "=" + readVersion );
}
fileIndex[i] = lv & 0xffffffffffffL;
}
// read some extra info from the end of the file, if present
long len = ra.length();
long pos = fileIndex[24];
int extraLen = 8 + 26*4;
if ( len == pos ) return; // old format o.k.
if ( len < pos+extraLen ) // > is o.k. for future extensions!
{
throw new IOException( "file of size " + len + " + too short, should be " + (pos+extraLen) );
}
ra.seek( pos );
ra.readFully( iobuffer, 0, extraLen );
dis = new ByteDataReader( iobuffer );
creationTime = dis.readLong();
if ( dis.readInt() != fileIndexCrc )
{
throw new IOException( "top index checksum error" );
}
fileHeaderCrcs = new int[25];
for( int i=0; i<25; i++ )
{
fileHeaderCrcs[i] = dis.readInt();
}
}
}