ben/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

statistical encoding

Browse source
This commit is contained in:
Arndt 2015-10-11 19:27:33 +02:00
parent f8dee5b7d1
commit ccf6641bad
41 changed files with 4543 additions and 1965 deletions
Download patch file Download diff file Expand all files Collapse all files

26
brouter-codec/pom.xml Normal file
View file

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.btools</groupId>
<artifactId>brouter</artifactId>
<version>1.2</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>brouter-codec</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.btools</groupId>
<artifactId>brouter-util</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

31
brouter-codec/src/main/java/btools/codec/DataBuffers.java Normal file
View file

@ -0,0 +1,31 @@
package btools.codec;
/**
* Container for some re-usable databuffers for the decoder
*/
public final class DataBuffers
{
public byte[] iobuffer;
public byte[] tagbuf1 = new byte[256];
public byte[] bbuf1 = new byte[65636];
public int[] ibuf1 = new int[4096];
public int[] ibuf2 = new int[2048];
public int[] ibuf3 = new int[2048];
public int[] alon = new int[2048];
public int[] alat = new int[2048];
public DataBuffers()
{
this( new byte[65636] );
}
/**
* construct a set of databuffers except
* for 'iobuffer', where the given array is used
*/
public DataBuffers( byte[] iobuffer )
{
this.iobuffer = iobuffer;
}
}

62
brouter-codec/src/main/java/btools/codec/IntegerFifo3Pass.java Normal file
View file

@ -0,0 +1,62 @@
package btools.codec;
/**
* Special integer fifo suitable for 3-pass encoding
*/
public class IntegerFifo3Pass
{
private int[] a;
private int size;
private int pos;
private int pass;
public IntegerFifo3Pass( int capacity )
{
a = capacity < 4 ? new int[4] : new int[capacity];
}
/**
* Starts a new encoding pass and resets the reading pointer
* from the stats collected in pass2 and writes that to the given context
*/
public void init()
{
pass++;
pos = 0;
}
/**
* writes to the fifo in pass2
*/
public void add( int value )
{
if ( pass == 2 )
{
if ( size == a.length )
{
int[] aa = new int[2 * size];
System.arraycopy( a, 0, aa, 0, size );
a = aa;
}
a[size++] = value;
}
}
/**
* reads from the fifo in pass3 (in pass1/2 returns just 1)
*/
public int getNext()
{
return pass == 3 ? get( pos++ ) : 1;
}
private int get( int idx )
{
if ( idx >= size )
{
throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
}
return a[idx];
}
}

87
brouter-codec/src/main/java/btools/codec/LinkedListContainer.java Normal file
View file

@ -0,0 +1,87 @@
package btools.codec;
/**
* Simple container for a list of lists of integers
*/
public class LinkedListContainer
{
private int[] ia; // prev, data, prev, data, ...
private int size;
private int[] startpointer; // 0=void, odd=head-data-cell
private int listpointer;
/**
* Construct a container for the given number of lists
*
* If no default-buffer is given, an int[nlists*4] is constructed,
* able to hold 2 entries per list on average
*
* @param nlists the number of lists
* @param defaultbuffer an optional data array for re-use (gets replaced if too small)
*/
public LinkedListContainer( int nlists, int[] defaultbuffer )
{
ia = defaultbuffer == null ? new int[nlists*4] : defaultbuffer;
startpointer = new int[nlists];
}
/**
* Add a data element to the given list
*
* @param listNr the list to add the data to
* @param data the data value
*/
public void addDataElement( int listNr, int data )
{
if ( size + 2 > ia.length )
{
resize();
}
ia[size++] = startpointer[ listNr ];
startpointer[ listNr ] = size;
ia[size++] = data;
}
/**
* Initialize a list for reading
*
* @param listNr the list to initialize
* @return the number of entries in that list
*/
public int initList( int listNr )
{
int cnt = 0;
int lp = listpointer = startpointer[ listNr ];
while( lp != 0 )
{
lp = ia[ lp-1 ];
cnt++;
}
return cnt;
}
/**
* Get a data element from the list previously initialized.
* Data elements are return in reverse order (lifo)
*
* @return the data element
* @throws IllegalArgumentException if no more element
*/
public int getDataElement()
{
if ( listpointer == 0 )
{
throw new IllegalArgumentException( "no more element!" );
}
int data = ia[ listpointer ];
listpointer = ia[ listpointer-1 ];
return data;
}
private void resize()
{
int[] ia2 = new int[2*ia.length];
System.arraycopy( ia, 0, ia2, 0, ia.length );
ia = ia2;
}
}

306
brouter-codec/src/main/java/btools/codec/MicroCache.java Normal file
View file

@ -0,0 +1,306 @@
package btools.codec;
import btools.util.ByteDataWriter;
/**
* a micro-cache is a data cache for an area of some square kilometers or some
* hundreds or thousands nodes
*
* This is the basic io-unit: always a full microcache is loaded from the
* data-file if a node is requested at a position not yet covered by the caches
* already loaded
*
* The nodes are represented in a compact way (typical 20-50 bytes per node),
* but in a way that they do not depend on each other, and garbage collection is
* supported to remove the nodes already consumed from the cache.
*
* The cache-internal data representation is different from that in the
* data-files, where a cache is encoded as a whole, allowing more
* redundancy-removal for a more compact encoding
*/
public class MicroCache extends ByteDataWriter
{
protected int[] faid;
protected int[] fapos;
protected int size = 0;
private int delcount = 0;
private int delbytes = 0;
private int p2size; // next power of 2 of size
// cache control: a virgin cache can be
// put to ghost state for later recovery
public boolean virgin = true;
public boolean ghost = false;
public static boolean debug = false;
protected MicroCache( byte[] ab )
{
super( ab );
}
public static MicroCache emptyCache()
{
return new MicroCache( null ); // TODO: singleton?
}
protected void init( int size )
{
this.size = size;
delcount = 0;
delbytes = 0;
p2size = 0x40000000;
while (p2size > size)
p2size >>= 1;
}
public void finishNode( long id )
{
fapos[size] = aboffset;
faid[size] = shrinkId( id );
size++;
}
public void discardNode()
{
aboffset = startPos( size );
}
public int getSize()
{
return size;
}
public int getDataSize()
{
return ab == null ? 0 : ab.length;
}
/**
* Set the internal reader (aboffset, aboffsetEnd) to the body data for the given id
*
* If a node is not found in an empty cache, this is usually an edge-effect
* (data-file does not exist or neighboured data-files of differnt age),
* but is can as well be a symptom of a node-identity breaking bug.
*
* Current implementation always returns false for not-found, however, for
* regression testing, at least for the case that is most likely a bug
* (node found but marked as deleted = ready for garbage collection
* = already consumed) the RunException should be re-enabled
*
* @return true if id was found
*/
public boolean getAndClear( long id64 )
{
if ( size == 0 )
{
return false;
}
int id = shrinkId( id64 );
int[] a = faid;
int offset = p2size;
int n = 0;
while (offset > 0)
{
int nn = n + offset;
if ( nn < size && a[nn] <= id )
{
n = nn;
}
offset >>= 1;
}
if ( a[n] == id )
{
if ( ( fapos[n] & 0x80000000 ) == 0 )
{
aboffset = startPos( n );
aboffsetEnd = fapos[n];
fapos[n] |= 0x80000000; // mark deleted
delbytes += aboffsetEnd - aboffset;
delcount++;
return true;
}
else // .. marked as deleted
{
// throw new RuntimeException( "MicroCache: node already consumed: id=" + id );
}
}
return false;
}
protected int startPos( int n )
{
return n > 0 ? fapos[n - 1] & 0x7fffffff : 0;
}
public void collect( int threshold )
{
if ( delcount > threshold )
{
virgin = false;
int nsize = size - delcount;
if ( nsize == 0 )
{
faid = null;
fapos = null;
}
else
{
int[] nfaid = new int[nsize];
int[] nfapos = new int[nsize];
int idx = 0;
byte[] nab = new byte[ab.length - delbytes];
int nab_off = 0;
for ( int i = 0; i < size; i++ )
{
int pos = fapos[i];
if ( ( pos & 0x80000000 ) == 0 )
{
int start = startPos( i );
int end = fapos[i];
int len = end - start;
System.arraycopy( ab, start, nab, nab_off, len );
nfaid[idx] = faid[i];
nab_off += len;
nfapos[idx] = nab_off;
idx++;
}
}
faid = nfaid;
fapos = nfapos;
ab = nab;
}
init( nsize );
}
}
public void unGhost()
{
ghost = false;
delcount = 0;
delbytes = 0;
for ( int i = 0; i < size; i++ )
{
fapos[i] &= 0x7fffffff; // clear deleted flags
}
}
/**
* @return the 64-bit global id for the given cache-position
*/
public long getIdForIndex( int i )
{
int id32 = faid[i];
return expandId( id32 );
}
/**
* expand a 32-bit micro-cache-internal id into a 64-bit (lon|lat) global-id
*
* @see #shrinkId
*/
public long expandId( int id32 )
{
throw new IllegalArgumentException( "expandId for empty cache" );
}
/**
* shrink a 64-bit (lon|lat) global-id into a a 32-bit micro-cache-internal id
*
* @see #expandId
*/
public int shrinkId( long id64 )
{
throw new IllegalArgumentException( "shrinkId for empty cache" );
}
/**
* @return true if the given lon/lat position is internal for that micro-cache
*/
public boolean isInternal( int ilon, int ilat )
{
throw new IllegalArgumentException( "isInternal for empty cache" );
}
/**
* (stasticially) encode the micro-cache into the format used in the datafiles
*
* @param buffer
* byte array to encode into (considered big enough)
* @return the size of the encoded data
*/
public int encodeMicroCache( byte[] buffer )
{
throw new IllegalArgumentException( "encodeMicroCache for empty cache" );
}
/**
* Compare the content of this microcache to another
*
* @return null if equals, else a diff-report
*/
public String compareWith( MicroCache mc )
{
String msg = _compareWith( mc );
if ( msg != null )
{
StringBuilder sb = new StringBuilder( msg );
sb.append( "\nencode cache:\n" ).append( summary() );
sb.append( "\ndecode cache:\n" ).append( mc.summary() );
return sb.toString();
}
return null;
}
private String summary()
{
StringBuilder sb = new StringBuilder( "size=" + size + " aboffset=" + aboffset );
for ( int i = 0; i < size; i++ )
{
sb.append( "\nidx=" + i + " faid=" + faid[i] + " fapos=" + fapos[i] );
}
return sb.toString();
}
private String _compareWith( MicroCache mc )
{
if ( size != mc.size )
{
return "size missmatch: " + size + "->" + mc.size;
}
for ( int i = 0; i < size; i++ )
{
if ( faid[i] != mc.faid[i] )
{
return "faid missmatch at index " + i + ":" + faid[i] + "->" + mc.faid[i];
}
int start = i > 0 ? fapos[i - 1] : 0;
int end = fapos[i] < mc.fapos[i] ? fapos[i] : mc.fapos[i];
int len = end - start;
for ( int offset = 0; offset < len; offset++ )
{
if ( mc.ab.length <= start + offset )
{
return "data buffer too small";
}
if ( ab[start + offset] != mc.ab[start + offset] )
{
return "data missmatch at index " + i + " offset=" + offset;
}
}
if ( fapos[i] != mc.fapos[i] )
{
return "fapos missmatch at index " + i + ":" + fapos[i] + "->" + mc.fapos[i];
}
}
if ( aboffset != mc.aboffset )
{
return "datasize missmatch: " + aboffset + "->" + mc.aboffset;
}
return null;
}
}

99
brouter-codec/src/main/java/btools/codec/MicroCache1.java Normal file
View file

@ -0,0 +1,99 @@
package btools.codec;
import btools.util.ByteDataWriter;
/**
* MicroCache1 is the old data format as of brouter 1.1 that does not allow to
* filter out unaccessable nodes at the beginning of the cache pipeline
*
* Kept for backward compatibility
*/
public final class MicroCache1 extends MicroCache
{
private int lonIdxBase;
private int latIdxBase;
public MicroCache1( int size, byte[] databuffer, int lonIdx80, int latIdx80 ) throws Exception
{
super( databuffer ); // sets ab=databuffer, aboffset=0
faid = new int[size];
fapos = new int[size];
this.size = 0;
lonIdxBase = ( lonIdx80 / 5 ) * 62500 + 31250;
latIdxBase = ( latIdx80 / 5 ) * 62500 + 31250;
}
public MicroCache1( byte[] databuffer, int lonIdx80, int latIdx80 ) throws Exception
{
super( databuffer ); // sets ab=databuffer, aboffset=0
lonIdxBase = ( lonIdx80 / 5 ) * 62500 + 31250;
latIdxBase = ( latIdx80 / 5 ) * 62500 + 31250;
size = readInt();
// get net size
int nbytes = 0;
for ( int i = 0; i < size; i++ )
{
aboffset += 4;
int bodySize = readVarLengthUnsigned();
aboffset += bodySize;
nbytes += bodySize;
}
// new array with only net data
byte[] nab = new byte[nbytes];
aboffset = 4;
int noffset = 0;
faid = new int[size];
fapos = new int[size];
for ( int i = 0; i < size; i++ )
{
faid[i] = readInt() ^ 0x8000; // flip lat-sign for correct ordering
int bodySize = readVarLengthUnsigned();
System.arraycopy( ab, aboffset, nab, noffset, bodySize );
aboffset += bodySize;
noffset += bodySize;
fapos[i] = noffset;
}
ab = nab;
aboffset = noffset;
init( size );
}
@Override
public long expandId( int id32 )
{
int lon32 = lonIdxBase + (short) ( id32 >> 16 );
int lat32 = latIdxBase + (short) ( ( id32 & 0xffff ) ^ 0x8000 );
return ( (long) lon32 ) << 32 | lat32;
}
@Override
public int shrinkId( long id64 )
{
int lon32 = (int) ( id64 >> 32 );
int lat32 = (int) ( id64 & 0xffffffff );
return ( lon32 - lonIdxBase ) << 16 | ( ( ( lat32 - latIdxBase ) & 0xffff ) ^ 0x8000 );
}
@Override
public int encodeMicroCache( byte[] buffer )
{
ByteDataWriter dos = new ByteDataWriter( buffer );
dos.writeInt( size );
for ( int n = 0; n < size; n++ )
{
dos.writeInt( faid[n] ^ 0x8000 );
int start = n > 0 ? fapos[n - 1] : 0;
int end = fapos[n];
int len = end - start;
dos.writeVarLengthUnsigned( len );
dos.write( ab, start, len );
}
return dos.size();
}
}

444
brouter-codec/src/main/java/btools/codec/MicroCache2.java Normal file
View file

@ -0,0 +1,444 @@
package btools.codec;
import java.util.BitSet;
import java.util.HashMap;
import btools.util.ByteArrayUnifier;
import btools.util.ByteDataReader;
/**
* MicroCache2 is the new format that uses statistical encoding and
* is able to do access filtering and waypoint matching during encoding
*/
public final class MicroCache2 extends MicroCache
{
private int lonBase;
private int latBase;
private int cellsize;
public MicroCache2( int size, byte[] databuffer, int lonIdx, int latIdx, int divisor ) throws Exception
{
super( databuffer ); // sets ab=databuffer, aboffset=0
faid = new int[size];
fapos = new int[size];
this.size = 0;
cellsize = 1000000 / divisor;
lonBase = lonIdx*cellsize;
latBase = latIdx*cellsize;
}
public byte[] readUnified( int len, ByteArrayUnifier u )
{
byte[] b = u.unify( ab, aboffset, len );
aboffset += len;
return b;
}
public MicroCache2( DataBuffers dataBuffers, int lonIdx, int latIdx, int divisor, TagValueValidator wayValidator, WaypointMatcher waypointMatcher ) throws Exception
{
super( null );
cellsize = 1000000 / divisor;
lonBase = lonIdx*cellsize;
latBase = latIdx*cellsize;
StatCoderContext bc = new StatCoderContext( dataBuffers.iobuffer );
TagValueCoder wayTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, wayValidator );
TagValueCoder nodeTagCoder = new TagValueCoder( bc, dataBuffers.tagbuf1, null );
NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder( bc );
NoisyDiffCoder nodeEleDiff = new NoisyDiffCoder( bc );
NoisyDiffCoder extLonDiff = new NoisyDiffCoder(bc);
NoisyDiffCoder extLatDiff = new NoisyDiffCoder(bc);
NoisyDiffCoder transEleDiff = new NoisyDiffCoder( bc );
size = bc.decodeNoisyNumber( 5 );
faid = size > dataBuffers.ibuf2.length ? new int[size] : dataBuffers.ibuf2;
fapos = size > dataBuffers.ibuf3.length ? new int[size] : dataBuffers.ibuf3;
int[] alon = size > dataBuffers.alon.length ? new int[size] : dataBuffers.alon;
int[] alat = size > dataBuffers.alat.length ? new int[size] : dataBuffers.alat;
if ( debug ) System.out.println( "*** decoding cache of size=" + size );
bc.decodeSortedArray( faid, 0, size, 0x20000000, 0 );
for( int n = 0; n<size; n++ )
{
long id64 = expandId( faid[n] );
alon[n] = (int)(id64 >> 32);
alat[n] = (int)(id64 & 0xffffffff);
}
int netdatasize = bc.decodeNoisyNumber( 10 );
ab = netdatasize > dataBuffers.bbuf1.length ? new byte[netdatasize] : dataBuffers.bbuf1;
aboffset = 0;
BitSet validNodes = new BitSet( size );
int finaldatasize = 0;
LinkedListContainer reverseLinks = new LinkedListContainer( size, dataBuffers.ibuf1 );
int selev = 0;
for( int n=0; n<size; n++ ) // loop over nodes
{
int ilon = alon[n];
int ilat = alat[n];
// future feature escape (turn restrictions?)
for(;;)
{
int featureId = bc.decodeVarBits();
if ( featureId == 0 ) break;
int bitsize = bc.decodeNoisyNumber( 5 );
for( int i=0; i< bitsize; i++ ) bc.decodeBit(); // just skip
}
selev += nodeEleDiff.decodeSignedValue();
writeShort( (short) selev );
writeVarBytes( nodeTagCoder.decodeTagValueSet() );
int links = bc.decodeNoisyNumber( 1 );
if ( debug ) System.out.println( "*** decoding node with links=" + links );
for( int li=0; li<links; li++ )
{
int startPointer = aboffset;
int sizeoffset = writeSizePlaceHolder();
int nodeIdx = n + nodeIdxDiff.decodeSignedValue();
int dlon_remaining;
int dlat_remaining;
boolean isReverse = false;
if ( nodeIdx != n ) // internal (forward-) link
{
writeVarLengthSigned( dlon_remaining = alon[nodeIdx] - ilon );
writeVarLengthSigned( dlat_remaining = alat[nodeIdx] - ilat );
}
else
{
isReverse = bc.decodeBit();
writeVarLengthSigned( dlon_remaining = extLonDiff.decodeSignedValue() );
writeVarLengthSigned( dlat_remaining = extLatDiff.decodeSignedValue() );
}
byte[] wayTags = wayTagCoder.decodeTagValueSet();
if ( wayTags != null )
{
validNodes.set( n, true ); // mark source-node valid
if ( nodeIdx != n ) // valid internal (forward-) link
{
reverseLinks.addDataElement( nodeIdx, n ); // register reverse link
finaldatasize += 1 + aboffset-startPointer; // reserve place for reverse
validNodes.set( nodeIdx, true ); // mark target-node valid
}
}
writeModeAndDesc( isReverse, wayTags );
if ( !isReverse ) // write geometry for forward links only
{
WaypointMatcher matcher = wayTags == null ? null : waypointMatcher;
if ( matcher != null ) matcher.startNode( ilon, ilat );
int ilontarget = ilon + dlon_remaining;
int ilattarget = ilat + dlat_remaining;
int transcount = bc.decodeVarBits();
if ( debug ) System.out.println( "*** decoding geometry with count=" + transcount );
int count = transcount+1;
for( int i=0; i<transcount; i++ )
{
int dlon = bc.decodePredictedValue( dlon_remaining/count );
int dlat = bc.decodePredictedValue( dlat_remaining/count );
dlon_remaining -= dlon;
dlat_remaining -= dlat;
count--;
writeVarLengthSigned( dlon );
writeVarLengthSigned( dlat );
writeVarLengthSigned( transEleDiff.decodeSignedValue() );
if ( matcher != null ) matcher.transferNode( ilontarget - dlon_remaining, ilattarget - dlat_remaining );
}
if ( matcher != null ) matcher.endNode( ilontarget, ilattarget );
}
if ( wayTags == null )
{
aboffset = startPointer; // not a valid link, delete it
}
else
{
injectSize( sizeoffset );
}
}
fapos[n] = aboffset;
}
// calculate final data size
int finalsize = 0;
for( int i=0; i<size; i++ )
{
int startpos = i > 0 ? fapos[i-1] : 0;
int endpos = fapos[i];
if ( validNodes.get( i ) )
{
finaldatasize += endpos-startpos;
finalsize++;
}
}
// append the reverse links at the end of each node
byte[] abOld = ab;
int[] faidOld = faid;
int[] faposOld = fapos;
int sizeOld = size;
ab = new byte[finaldatasize];
faid = new int[finalsize];
fapos = new int[finalsize];
aboffset = 0;
size = 0;
for( int n=0; n<sizeOld; n++ )
{
if ( !validNodes.get( n ) )
{
continue;
}
int startpos = n > 0 ? faposOld[n-1] : 0;
int endpos = faposOld[n];
int len = endpos-startpos;
System.arraycopy( abOld, startpos, ab, aboffset, len );
if ( debug ) System.out.println( "*** copied " + len + " bytes from " + aboffset + " for node " + n );
aboffset += len;
int cnt = reverseLinks.initList( n );
if ( debug ) System.out.println( "*** appending " + cnt + " reverse links for node " + n );
for( int ri = 0; ri < cnt; ri++ )
{
int nodeIdx = reverseLinks.getDataElement();
int sizeoffset = writeSizePlaceHolder();
writeVarLengthSigned( alon[nodeIdx] - alon[n] );
writeVarLengthSigned( alat[nodeIdx] - alat[n] );
writeModeAndDesc( true, null );
injectSize( sizeoffset );
}
faid[size] = faidOld[n];
fapos[size] = aboffset;
size++;
}
init( size );
}
@Override
public long expandId( int id32 )
{
int dlon = 0;
int dlat = 0;
for( int bm = 1; bm < 0x8000; bm <<= 1 )
{
if ( (id32 & 1) != 0 ) dlon |= bm;
if ( (id32 & 2) != 0 ) dlat |= bm;
id32 >>= 2;
}
int lon32 = lonBase + dlon;
int lat32 = latBase + dlat;
return ((long)lon32)<<32 | lat32;
}
@Override
public int shrinkId( long id64 )
{
int lon32 = (int)(id64 >> 32);
int lat32 = (int)(id64 & 0xffffffff);
int dlon = lon32 - lonBase;
int dlat = lat32 - latBase;
int id32 = 0;
for( int bm = 0x4000; bm > 0; bm >>= 1 )
{
id32 <<= 2;
if ( ( dlon & bm ) != 0 ) id32 |= 1;
if ( ( dlat & bm ) != 0 ) id32 |= 2;
}
return id32;
}
@Override
public boolean isInternal( int ilon, int ilat )
{
return ilon >= lonBase && ilon < lonBase + cellsize
&& ilat >= latBase && ilat < latBase + cellsize;
}
@Override
public int encodeMicroCache( byte[] buffer )
{
HashMap<Long,Integer> idMap = new HashMap<Long,Integer>();
for( int n=0; n<size; n++ ) // loop over nodes
{
idMap.put( Long.valueOf( expandId( faid[n] ) ), Integer.valueOf( n ) );
}
IntegerFifo3Pass linkCounts = new IntegerFifo3Pass( 256 );
IntegerFifo3Pass transCounts = new IntegerFifo3Pass( 256 );
TagValueCoder wayTagCoder = new TagValueCoder();
TagValueCoder nodeTagCoder = new TagValueCoder();
NoisyDiffCoder nodeIdxDiff = new NoisyDiffCoder();
NoisyDiffCoder nodeEleDiff = new NoisyDiffCoder();
NoisyDiffCoder extLonDiff = new NoisyDiffCoder();
NoisyDiffCoder extLatDiff = new NoisyDiffCoder();
NoisyDiffCoder transEleDiff = new NoisyDiffCoder();
int netdatasize = 0;
for(int pass=1;; pass++) // 3 passes: counters, stat-collection, encoding
{
boolean dostats = pass == 3;
boolean dodebug = debug && pass == 3;
if ( pass < 3 ) netdatasize = fapos[size-1];
StatCoderContext bc = new StatCoderContext( buffer );
linkCounts.init();
transCounts.init();
wayTagCoder.encodeDictionary( bc );
if ( dostats ) bc.assignBits( "wayTagDictionary" );
nodeTagCoder.encodeDictionary( bc );
if ( dostats ) bc.assignBits( "nodeTagDictionary" );
nodeIdxDiff.encodeDictionary( bc );
nodeEleDiff.encodeDictionary( bc );
extLonDiff.encodeDictionary( bc );
extLatDiff.encodeDictionary( bc );
transEleDiff.encodeDictionary( bc );
if ( dostats ) bc.assignBits( "noisebits" );
bc.encodeNoisyNumber( size, 5 );
if ( dostats ) bc.assignBits( "nodecount" );
bc.encodeSortedArray( faid, 0, size, 0x20000000, 0 );
if ( dostats ) bc.assignBits( "node-positions" );
bc.encodeNoisyNumber( netdatasize, 10 ); // net-size
if ( dostats ) bc.assignBits( "netdatasize" );
if ( dodebug ) System.out.println( "*** encoding cache of size=" + size );
int lastSelev = 0;
for( int n=0; n<size; n++ ) // loop over nodes
{
aboffset = startPos( n );
aboffsetEnd = fapos[n];
if ( dodebug ) System.out.println( "*** encoding node " + n + " from " + aboffset + " to " + aboffsetEnd );
// future feature escape (turn restrictions?)
bc.encodeVarBits( 0 );
int selev = readShort();
nodeEleDiff.encodeSignedValue( selev - lastSelev );
if ( dostats ) bc.assignBits( "nodeele" );
lastSelev = selev;
nodeTagCoder.encodeTagValueSet( readVarBytes() );
if ( dostats ) bc.assignBits( "nodeTagIdx" );
int nlinks = linkCounts.getNext();
if ( dodebug ) System.out.println( "*** nlinks=" + nlinks );
bc.encodeNoisyNumber( nlinks, 1 );
if ( dostats ) bc.assignBits( "link-counts" );
long id64 = expandId( faid[n] );
int ilon = (int)(id64 >> 32);
int ilat = (int)(id64 & 0xffffffff);
nlinks = 0;
while( hasMoreData() ) // loop over links
{
// read link data
int startPointer = aboffset;
int endPointer = getEndPointer();
int ilonlink = ilon + readVarLengthSigned();
int ilatlink = ilat + readVarLengthSigned();
int sizecode = readVarLengthUnsigned();
boolean isReverse = ( sizecode & 1 ) != 0;
int descSize = sizecode >> 1;
byte[] description = null;
if ( descSize > 0 )
{
description = new byte[descSize];
readFully( description );
}
boolean isInternal = isInternal( ilonlink, ilatlink );
if ( isReverse && isInternal )
{
if ( dodebug ) System.out.println( "*** NOT encoding link reverse=" + isReverse + " internal=" + isInternal );
netdatasize -= aboffset-startPointer;
continue; // do not encode internal reverse links
}
if ( dodebug ) System.out.println( "*** encoding link reverse=" + isReverse + " internal=" + isInternal );
nlinks++;
if ( isInternal )
{
long link64 = ((long)ilonlink)<<32 | ilatlink;
Integer idx = idMap.get( Long.valueOf( link64 ) );
if ( idx == null ) throw new RuntimeException( "ups: internal not found?" );
int nodeIdx = idx.intValue();
if ( dodebug ) System.out.println( "*** target nodeIdx=" + nodeIdx );
if ( nodeIdx == n ) throw new RuntimeException( "ups: self ref?" );
nodeIdxDiff.encodeSignedValue( nodeIdx - n );
if ( dostats ) bc.assignBits( "nodeIdx" );
}
else
{
nodeIdxDiff.encodeSignedValue( 0 );
bc.encodeBit( isReverse );
extLonDiff.encodeSignedValue( ilonlink - ilon );
extLatDiff.encodeSignedValue( ilatlink - ilat );
if ( dostats ) bc.assignBits( "externalNode" );
}
wayTagCoder.encodeTagValueSet( description );
if ( dostats ) bc.assignBits( "wayDescIdx" );
if ( !isReverse )
{
byte[] geometry = readDataUntil( endPointer );
// write transition nodes
int count = transCounts.getNext();
if ( dodebug ) System.out.println( "*** encoding geometry with count=" + count );
bc.encodeVarBits( count++ );
if ( dostats ) bc.assignBits( "transcount" );
int transcount = 0;
if ( geometry != null )
{
int dlon_remaining = ilonlink - ilon;
int dlat_remaining = ilatlink - ilat;
ByteDataReader r = new ByteDataReader( geometry );
while ( r.hasMoreData() )
{
transcount++;
int dlon = r.readVarLengthSigned();
int dlat = r.readVarLengthSigned();
bc.encodePredictedValue( dlon, dlon_remaining/count );
bc.encodePredictedValue( dlat, dlat_remaining/count );
dlon_remaining -= dlon;
dlat_remaining -= dlat;
if ( count > 1 ) count--;
if ( dostats ) bc.assignBits( "transpos" );
transEleDiff.encodeSignedValue( r.readVarLengthSigned() );
if ( dostats ) bc.assignBits( "transele" );
}
}
transCounts.add( transcount );
}
}
linkCounts.add( nlinks );
}
if ( pass == 3 )
{
return bc.getEncodedLength();
}
}
}
}

92
brouter-codec/src/main/java/btools/codec/NoisyDiffCoder.java Normal file
View file

@ -0,0 +1,92 @@
package btools.codec;
/**
* Encoder/Decoder for signed integers that automatically detects the typical
* range of these numbers to determine a noisy-bit count as a very simple
* dictionary
*
* Adapted for 3-pass encoding (counters -> statistics -> encoding )
* but doesn't do anything at pass1
*/
public final class NoisyDiffCoder
{
private int tot;
private int[] freqs;
private int noisybits;
private StatCoderContext bc;
private int pass;
/**
* Create a decoder and read the noisy-bit count from the gibe context
*/
public NoisyDiffCoder( StatCoderContext bc )
{
noisybits = bc.decodeVarBits();
this.bc = bc;
}
/**
* Create an encoder for 3-pass-encoding
*/
public NoisyDiffCoder()
{
}
/**
* encodes a signed int (pass3 only, stats collection in pass2)
*/
public void encodeSignedValue( int value )
{
if ( pass == 3 )
{
bc.encodeNoisyDiff( value, noisybits );
}
else if ( pass == 2 )
{
count( value < 0 ? -value : value );
}
}
/**
* decodes a signed int
*/
public int decodeSignedValue()
{
return bc.decodeNoisyDiff( noisybits );
}
/**
* Starts a new encoding pass and (in pass3) calculates the noisy-bit count
* from the stats collected in pass2 and writes that to the given context
*/
public void encodeDictionary( StatCoderContext bc )
{
if ( ++pass == 3 )
{
// how many noisy bits?
for ( noisybits = 0; noisybits < 14 && tot > 0; noisybits++ )
{
if ( freqs[noisybits] < ( tot >> 1 ) )
break;
}
bc.encodeVarBits( noisybits );
}
this.bc = bc;
}
private void count( int value )
{
if ( freqs == null )
freqs = new int[14];
int bm = 1;
for ( int i = 0; i < 14; i++ )
{
if ( value < bm )
break;
else
freqs[i]++;
bm <<= 1;
}
tot++;
}
}

291
brouter-codec/src/main/java/btools/codec/StatCoderContext.java Normal file
View file

@ -0,0 +1,291 @@
package btools.codec;
import java.util.TreeMap;
import btools.util.BitCoderContext;
public final class StatCoderContext extends BitCoderContext
{
private static TreeMap<String, long[]> statsPerName;
private long lastbitpos = 0;
public StatCoderContext( byte[] ab )
{
super( ab );
}
/**
* assign the de-/encoded bits since the last call assignBits to the given
* name. Used for encoding statistics
*
* @see #getBitReport
*/
public void assignBits( String name )
{
long bitpos = getBitPosition();
if ( statsPerName == null )
{
statsPerName = new TreeMap<String, long[]>();
}
long[] stats = statsPerName.get( name );
if ( stats == null )
{
stats = new long[2];
statsPerName.put( name, stats );
}
stats[0] += bitpos - lastbitpos;
stats[1] += 1;
lastbitpos = bitpos;
}
/**
* Get a textual report on the bit-statistics
*
* @see #assignBits
*/
public static String getBitReport()
{
StringBuilder sb = new StringBuilder();
for ( String name : statsPerName.keySet() )
{
long[] stats = statsPerName.get( name );
sb.append( name + " count=" + stats[1] + " bits=" + stats[0] + "\n" );
}
statsPerName = null;
return sb.toString();
}
/**
* encode an unsigned integer with some of of least significant bits
* considered noisy
*
* @see #decodeNoisyNumber
*/
public void encodeNoisyNumber( int value, int noisybits )
{
if ( value < 0 )
{
throw new IllegalArgumentException( "encodeVarBits expects positive value" );
}
if ( noisybits > 0 )
{
int mask = 0xffffffff >>> ( 32 - noisybits );
encodeBounded( mask, value & mask );
value >>= noisybits;
}
encodeVarBits( value );
}
/**
* decode an unsigned integer with some of of least significant bits
* considered noisy
*
* @see #encodeNoisyNumber
*/
public int decodeNoisyNumber( int noisybits )
{
int value = 0;
if ( noisybits > 0 )
{
int mask = 0xffffffff >>> ( 32 - noisybits );
value = decodeBounded( mask );
}
return value | ( decodeVarBits() << noisybits );
}
/**
* encode a signed integer with some of of least significant bits considered
* noisy
*
* @see #decodeNoisyDiff
*/
public void encodeNoisyDiff( int value, int noisybits )
{
if ( noisybits > 0 )
{
value += 1 << ( noisybits - 1 );
int mask = 0xffffffff >>> ( 32 - noisybits );
encodeBounded( mask, value & mask );
value >>= noisybits;
}
encodeVarBits( value < 0 ? -value : value );
if ( value != 0 )
{
encodeBit( value < 0 );
}
}
/**
* decode a signed integer with some of of least significant bits considered
* noisy
*
* @see #encodeNoisyDiff
*/
public int decodeNoisyDiff( int noisybits )
{
int value = 0;
if ( noisybits > 0 )
{
int mask = 0xffffffff >>> ( 32 - noisybits );
value = decodeBounded( mask ) - ( 1 << ( noisybits - 1 ) );
}
int val2 = decodeVarBits() << noisybits;
if ( val2 != 0 )
{
if ( decodeBit() )
{
val2 = -val2;
}
}
return value + val2;
}
/**
* encode a signed integer with the typical range and median taken from the
* predicted value
*
* @see #decodePredictedValue
*/
public void encodePredictedValue( int value, int predictor )
{
int p = predictor < 0 ? -predictor : predictor;
int noisybits = 0;
while (p > 2)
{
noisybits++;
p >>= 1;
}
encodeNoisyDiff( value - predictor, noisybits );
}
/**
* decode a signed integer with the typical range and median taken from the
* predicted value
*
* @see #encodePredictedValue
*/
public int decodePredictedValue( int predictor )
{
int p = predictor < 0 ? -predictor : predictor;
int noisybits = 0;
while (p > 2)
{
noisybits++;
p >>= 1;
}
return predictor + decodeNoisyDiff( noisybits );
}
/**
* encode an integer-array making use of the fact that it is sorted. This is
* done, starting with the most significant bit, by recursively encoding the
* number of values with the current bit being 0. This yields an number of
* bits per value that only depends on the typical distance between subsequent
* values and also benefits
*
* @param values
* the array to encode
* @param offset
* position in this array where to start
* @param subsize
* number of values to encode
* @param nextbit
* bitmask with the most significant bit set to 1
* @param mask
* should be 0
*/
public void encodeSortedArray( int[] values, int offset, int subsize, int nextbit, int mask )
{
if ( subsize == 1 ) // last-choice shortcut
{
while (nextbit != 0)
{
encodeBit( ( values[offset] & nextbit ) != 0 );
nextbit >>= 1;
}
}
if ( nextbit == 0 )
{
return;
}
int data = mask & values[offset];
mask |= nextbit;
// count 0-bit-fraction
int i = offset;
int end = subsize + offset;
for ( ; i < end; i++ )
{
if ( ( values[i] & mask ) != data )
{
break;
}
}
int size1 = i - offset;
int size2 = subsize - size1;
encodeBounded( subsize, size1 );
if ( size1 > 0 )
{
encodeSortedArray( values, offset, size1, nextbit >> 1, mask );
}
if ( size2 > 0 )
{
encodeSortedArray( values, i, size2, nextbit >> 1, mask );
}
}
/**
* @see #encodeSortedArray
*
* @param values
* the array to encode
* @param offset
* position in this array where to start
* @param subsize
* number of values to encode
* @param nextbit
* bitmask with the most significant bit set to 1
* @param value
* should be 0
*/
public void decodeSortedArray( int[] values, int offset, int subsize, int nextbit, int value )
{
if ( subsize == 1 ) // last-choice shortcut
{
while (nextbit != 0)
{
if ( decodeBit() )
{
value |= nextbit;
}
nextbit >>= 1;
}
values[offset] = value;
return;
}
if ( nextbit == 0 )
{
while (subsize-- > 0)
{
values[offset++] = value;
}
return;
}
int size1 = decodeBounded( subsize );
int size2 = subsize - size1;
if ( size1 > 0 )
{
decodeSortedArray( values, offset, size1, nextbit >> 1, value );
}
if ( size2 > 0 )
{
decodeSortedArray( values, offset + size1, size2, nextbit >> 1, value | nextbit );
}
}
}

235
brouter-codec/src/main/java/btools/codec/TagValueCoder.java Normal file
View file

@ -0,0 +1,235 @@
package btools.codec;
import java.util.HashMap;
import java.util.PriorityQueue;
import btools.util.BitCoderContext;
/**
* Encoder/Decoder for way-/node-descriptions
*
* It detects identical descriptions and sorts them
* into a huffman-tree according to their frequencies
*
* Adapted for 3-pass encoding (counters -> statistics -> encoding )
* but doesn't do anything at pass1
*/
public final class TagValueCoder
{
private HashMap<TagValueSet, TagValueSet> identityMap;
private Object tree;
private BitCoderContext bc;
private int pass;
public void encodeTagValueSet( byte[] data )
{
if ( pass == 1 )
{
return;
}
TagValueSet tvsProbe = new TagValueSet();
tvsProbe.data = data;
TagValueSet tvs = identityMap.get( tvsProbe );
if ( pass == 3 )
{
bc.encodeBounded( tvs.range - 1, tvs.code );
}
else if ( pass == 2 )
{
if ( tvs == null )
{
tvs = tvsProbe;
identityMap.put( tvs, tvs );
}
tvs.frequency++;
}
}
public byte[] decodeTagValueSet()
{
Object node = tree;
while (node instanceof TreeNode)
{
TreeNode tn = (TreeNode) node;
boolean nextBit = bc.decodeBit();
node = nextBit ? tn.child2 : tn.child1;
}
return (byte[]) node;
}
public void encodeDictionary( BitCoderContext bc )
{
if ( ++pass == 3 )
{
PriorityQueue<TagValueSet> queue = new PriorityQueue<TagValueSet>( identityMap.values() );
while (queue.size() > 1)
{
TagValueSet node = new TagValueSet();
node.child1 = queue.poll();
node.child2 = queue.poll();
node.frequency = node.child1.frequency + node.child2.frequency;
queue.add( node );
}
TagValueSet root = queue.poll();
root.encode( bc, 1, 0 );
}
this.bc = bc;
}
public TagValueCoder( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
{
tree = decodeTree( bc, buffer, validator );
this.bc = bc;
}
public TagValueCoder()
{
identityMap = new HashMap<TagValueSet, TagValueSet>();
}
private Object decodeTree( BitCoderContext bc, byte[] buffer, TagValueValidator validator )
{
boolean isNode = bc.decodeBit();
if ( isNode )
{
TreeNode node = new TreeNode();
node.child1 = decodeTree( bc, buffer, validator );
node.child2 = decodeTree( bc, buffer, validator );
return node;
}
BitCoderContext target = null;
for ( ;; )
{
int delta = bc.decodeVarBits();
if ( target == null )
{
if ( delta == 0 )
return null;
target = new BitCoderContext( buffer );
target.encodeBit( false ); // dummy reverse bit
}
target.encodeVarBits( delta );
if ( delta == 0 )
break;
int data = bc.decodeVarBits();
target.encodeVarBits( data );
}
int len = target.getEncodedLength();
byte[] res = new byte[len];
System.arraycopy( buffer, 0, res, 0, len );
if ( validator == null || validator.accessAllowed( res ) )
{
return res;
}
return null;
}
public static final class TreeNode
{
public Object child1;
public Object child2;
}
public static final class TagValueSet implements Comparable<TagValueSet>
{
public byte[] data;
public int frequency;
public int code;
public int range;
public TagValueSet child1;
public TagValueSet child2;
public void encode( BitCoderContext bc, int range, int code )
{
this.range = range;
this.code = code;
boolean isNode = child1 != null;
bc.encodeBit( isNode );
if ( isNode )
{
child1.encode( bc, range << 1, code );
child2.encode( bc, range << 1, code + range );
}
else
{
if ( data == null )
{
bc.encodeVarBits( 0 );
return;
}
BitCoderContext src = new BitCoderContext( data );
if ( src.decodeBit() )
{
throw new IllegalArgumentException( "cannot encode reverse bit!" );
}
for ( ;; )
{
int delta = src.decodeVarBits();
bc.encodeVarBits( delta );
if ( delta == 0 )
{
break;
}
int data = src.decodeVarBits();
bc.encodeVarBits( data );
}
}
}
@Override
public boolean equals( Object o )
{
if ( o instanceof TagValueSet )
{
TagValueSet tvs = (TagValueSet) o;
if ( data == null )
{
return tvs.data == null;
}
if ( tvs.data == null )
{
return data == null;
}
if ( data.length != tvs.data.length )
{
return false;
}
for ( int i = 0; i < data.length; i++ )
{
if ( data[i] != tvs.data[i] )
{
return false;
}
}
return true;
}
return false;
}
@Override
public int hashCode()
{
if ( data == null )
{
return 0;
}
int h = 17;
for ( int i = 0; i < data.length; i++ )
{
h = ( h << 8 ) + data[i];
}
return h;
}
@Override
public int compareTo( TagValueSet tvs )
{
if ( frequency < tvs.frequency )
return -1;
if ( frequency > tvs.frequency )
return 1;
return 0;
}
}
}

11
brouter-codec/src/main/java/btools/codec/TagValueValidator.java Normal file
View file

@ -0,0 +1,11 @@
package btools.codec;
public interface TagValueValidator
{
/**
* @param tagValueSet the way description to check
* @return true if access is allowed in the current profile
*/
public boolean accessAllowed( byte[] tagValueSet );
}

13
brouter-codec/src/main/java/btools/codec/WaypointMatcher.java Normal file
View file

@ -0,0 +1,13 @@
package btools.codec;
/**
* a waypoint matcher gets way geometries
* from the decoder to find the closest
* matches to the waypoints
*/
public interface WaypointMatcher
{
void startNode( int ilon, int ilat );
void transferNode( int ilon, int ilat );
void endNode( int ilon, int ilat );
}

52
brouter-codec/src/test/java/btools/codec/LinkedListContainerTest.java Normal file
View file

@ -0,0 +1,52 @@
package btools.codec;
import org.junit.Assert;
import org.junit.Test;
public class LinkedListContainerTest
{
@Test
public void linkedListTest1()
{
int nlists = 553;
LinkedListContainer llc = new LinkedListContainer( nlists, null );
for ( int ln = 0; ln < nlists; ln++ )
{
for ( int i = 0; i < 10; i++ )
{
llc.addDataElement( ln, ln * i );
}
}
for ( int i = 0; i < 10; i++ )
{
for ( int ln = 0; ln < nlists; ln++ )
{
llc.addDataElement( ln, ln * i );
}
}
for ( int ln = 0; ln < nlists; ln++ )
{
int cnt = llc.initList( ln );
Assert.assertTrue( "list size test", cnt == 20 );
for ( int i = 19; i >= 0; i-- )
{
int data = llc.getDataElement();
Assert.assertTrue( "data value test", data == ln * ( i % 10 ) );
}
}
try
{
llc.getDataElement();
Assert.fail( "no more elements expected" );
}
catch (IllegalArgumentException e)
{
}
}
}

127
brouter-codec/src/test/java/btools/codec/StatCoderContextTest.java Normal file
View file

@ -0,0 +1,127 @@
package btools.codec;
import java.util.Arrays;
import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
public class StatCoderContextTest
{
@Test
public void noisyVarBitsEncodeDecodeTest()
{
byte[] ab = new byte[40000];
StatCoderContext ctx = new StatCoderContext( ab );
for ( int noisybits = 0; noisybits < 12; noisybits++ )
{
for ( int i = 0; i < 1000; i++ )
{
ctx.encodeNoisyNumber( i, noisybits );
}
}
ctx = new StatCoderContext( ab );
for ( int noisybits = 0; noisybits < 12; noisybits++ )
{
for ( int i = 0; i < 1000; i++ )
{
int value = ctx.decodeNoisyNumber( noisybits );
if ( value != i )
{
Assert.fail( "value mismatch: noisybits=" + noisybits + " i=" + i + " value=" + value );
}
}
}
}
@Test
public void noisySignedVarBitsEncodeDecodeTest()
{
byte[] ab = new byte[80000];
StatCoderContext ctx = new StatCoderContext( ab );
for ( int noisybits = 0; noisybits < 12; noisybits++ )
{
for ( int i = -1000; i < 1000; i++ )
{
ctx.encodeNoisyDiff( i, noisybits );
}
}
ctx = new StatCoderContext( ab );
for ( int noisybits = 0; noisybits < 12; noisybits++ )
{
for ( int i = -1000; i < 1000; i++ )
{
int value = ctx.decodeNoisyDiff( noisybits );
if ( value != i )
{
Assert.fail( "value mismatch: noisybits=" + noisybits + " i=" + i + " value=" + value );
}
}
}
}
@Test
public void predictedValueEncodeDecodeTest()
{
byte[] ab = new byte[80000];
StatCoderContext ctx = new StatCoderContext( ab );
for ( int value = -100; value < 100; value += 5 )
{
for ( int predictor = -200; predictor < 200; predictor += 7 )
{
ctx.encodePredictedValue( value, predictor );
}
}
ctx = new StatCoderContext( ab );
for ( int value = -100; value < 100; value += 5 )
{
for ( int predictor = -200; predictor < 200; predictor += 7 )
{
int decodedValue = ctx.decodePredictedValue( predictor );
if ( value != decodedValue )
{
Assert.fail( "value mismatch: value=" + value + " predictor=" + predictor + " decodedValue=" + decodedValue );
}
}
}
}
@Test
public void sortedArrayEncodeDecodeTest()
{
Random rand = new Random();
int size = 1000000;
int[] values = new int[size];
for ( int i = 0; i < size; i++ )
{
values[i] = rand.nextInt() & 0x0fffffff;
}
values[5] = 175384; // force collision
values[8] = 175384;
values[15] = 275384; // force neighbours
values[18] = 275385;
Arrays.sort( values );
byte[] ab = new byte[3000000];
StatCoderContext ctx = new StatCoderContext( ab );
ctx.encodeSortedArray( values, 0, size, 0x08000000, 0 );
ctx = new StatCoderContext( ab );
int[] decodedValues = new int[size];
ctx.decodeSortedArray( decodedValues, 0, size, 0x08000000, 0 );
for ( int i = 0; i < size; i++ )
{
if ( values[i] != decodedValues[i] )
{
Assert.fail( "mismatch at i=" + i + " " + values[i] + "<>" + decodedValues[i] );
}
}
}
}