added crc logic for datafile access
This commit is contained in:
Arndt Brenschede
parent
e88465f5ec
commit
b4aa961f2d
26 changed files with 621 additions and 1364 deletions
|
|
@ -1,328 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Memory efficient Map to map a long-key to an object-value
|
||||
*
|
||||
* Implementation is such that basically the 12 bytes
|
||||
* per entry is allocated that's needed to store
|
||||
* a long- and an object-value.
|
||||
* This class does not implement the Map interface
|
||||
* because it's not complete (remove() is not implemented,
|
||||
* CompactLongMap can only grow.)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongMap<V>
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
private V value_in;
|
||||
protected V value_out;
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongMap()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new Object[MAXLISTS][];
|
||||
vla[0] = new Object[1];
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
* The Map extension:
|
||||
* next 5 protected methods are needed to implement value-support
|
||||
* overwrite them all to support value structures other than the
|
||||
* long-values implemented here as a sample.
|
||||
*
|
||||
* Furthermore, put() and get() method need to be implemented
|
||||
* to access the values.
|
||||
*
|
||||
* Note that this map does not behave exactly like java.util.Map
|
||||
* - put(..) with already existing key throws exception
|
||||
* - get(..) with non-existing key thros exception
|
||||
*
|
||||
* If you have keys that cannot easily be mapped on long's, use
|
||||
* a hash-function to do the mapping. But note that, in comparison
|
||||
* to java.util.HashMap, in that case the keys itself are not saved,
|
||||
* only the hash-values, so you need to be sure that random duplicate
|
||||
* hashs are either excluded by the structure of your data or that
|
||||
* you can handle the possible IllegalArgumentException
|
||||
*
|
||||
*/
|
||||
|
||||
private Object[][] vla; // value list array
|
||||
|
||||
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
try
|
||||
{
|
||||
value_in = value;
|
||||
if ( contains( id, true ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_in = null;
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as put( id, value ) but duplicate check
|
||||
* is skipped for performance. Be aware that you
|
||||
* can get a duplicate exception later on if the
|
||||
* map is restructured!
|
||||
* with System parameter earlyDuplicateCheck=true you
|
||||
* can enforce the early duplicate check for debugging
|
||||
*
|
||||
* @param id the key to insert
|
||||
* @param value the value to insert object
|
||||
* @exception IllegalArgumentException for duplicates if enabled
|
||||
*/
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
if ( earlyDuplicateCheck && contains( id ) )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in early check: " + id );
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value for the given id
|
||||
* @param id the key to query
|
||||
* @return the object, or null if id not known
|
||||
*/
|
||||
public V get( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
if ( contains( id, false ) )
|
||||
{
|
||||
return value_out;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this map
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new Object[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
return contains( id, false );
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, doPut ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, boolean doPut )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = (V)vla[idx][n];
|
||||
if ( doPut ) vla[idx][n] = value_in;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArrays( long[] faid, ArrayList<V> flv )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
flv.add( (V)vla[minIdx][pa[minIdx]] );
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen arrays
|
||||
al = null;
|
||||
vla = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Memory efficient Set for long-keys
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongSet
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongSet()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* add a long value to this set if not yet in.
|
||||
* @param id the value to add to this set.
|
||||
* @return true if "id" already contained in this set.
|
||||
*/
|
||||
public boolean add( long id )
|
||||
{
|
||||
if ( contains( id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
_add( id );
|
||||
}
|
||||
|
||||
private void _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArray( long[] faid )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen array
|
||||
al = null;
|
||||
}
|
||||
|
||||
}
|
||||
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 0, 1 );
|
||||
hashMapComparison( 1, 1 );
|
||||
hashMapComparison( 2, 2 );
|
||||
hashMapComparison( 3, 3 );
|
||||
hashMapComparison( 4, 4 );
|
||||
hashMapComparison( 5, 5 );
|
||||
hashMapComparison( 7, 10 );
|
||||
hashMapComparison( 8, 10 );
|
||||
hashMapComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,String> hmap = new HashMap<Long,String>();
|
||||
CompactLongMap<String> cmap_slow = new CompactLongMap<String>();
|
||||
CompactLongMap<String> cmap_fast = new CompactLongMap<String>();
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
String s = "" + i;
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hmap.containsKey( KK ) )
|
||||
{
|
||||
hmap.put( KK, s );
|
||||
cmap_slow.put( k, s );
|
||||
cmap_fast.fastPut( k, s );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cmap_slow = new FrozenLongMap<String>( cmap_slow );
|
||||
cmap_fast = new FrozenLongMap<String>( cmap_fast );
|
||||
}
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
String s = hmap.get( KK );
|
||||
|
||||
boolean contained = hmap.containsKey( KK );
|
||||
Assert.assertTrue( "containsKey missmatch (slow)", contained == cmap_slow.contains( k ) );
|
||||
Assert.assertTrue( "containsKey missmatch (fast)", contained == cmap_fast.contains( k ) );
|
||||
|
||||
if ( contained )
|
||||
{
|
||||
Assert.assertEquals( "object missmatch (fast)", s, cmap_fast.get( k ) );
|
||||
Assert.assertEquals( "object missmatch (slow)", s, cmap_slow.get( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactSetTest
|
||||
{
|
||||
@Test
|
||||
public void hashSetComparisonTest()
|
||||
{
|
||||
hashSetComparison( 0, 1 );
|
||||
hashSetComparison( 1, 1 );
|
||||
hashSetComparison( 2, 2 );
|
||||
hashSetComparison( 3, 3 );
|
||||
hashSetComparison( 4, 4 );
|
||||
hashSetComparison( 5, 5 );
|
||||
hashSetComparison( 7, 10 );
|
||||
hashSetComparison( 8, 10 );
|
||||
hashSetComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashSetComparison( int setsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
CompactLongSet cset_slow = new CompactLongSet();
|
||||
CompactLongSet cset_fast = new CompactLongSet();
|
||||
|
||||
for( int i=0; i<setsize; i++ )
|
||||
{
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hset.contains( KK ) )
|
||||
{
|
||||
hset.add( KK );
|
||||
cset_slow.add( k );
|
||||
cset_fast.fastAdd( k );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cset_slow = new FrozenLongSet( cset_slow );
|
||||
cset_fast = new FrozenLongSet( cset_fast );
|
||||
}
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
boolean contained = hset.contains( KK );
|
||||
Assert.assertTrue( "contains missmatch (slow)", contained == cset_slow.contains( k ) );
|
||||
Assert.assertTrue( "contains missmatch (fast)", contained == cset_fast.contains( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Special Memory efficient Map to map a long-key to
|
||||
* a "small" value (some bits only) where it is expected
|
||||
* that the keys are dense, so that we can use more or less
|
||||
* a simple array as the best-fit data model (except for
|
||||
* the 32-bit limit of arrays!)
|
||||
*
|
||||
* Additionally, to enable small-memory unit testing
|
||||
* of code using this map this one has a fallback for a small map
|
||||
* where we have only few keys, but not dense. In this
|
||||
* case, we use the mechanics of the CompactLongMap
|
||||
*
|
||||
* Target application are osm-node ids which are in the
|
||||
* range 0...3 billion and basically dense (=only few
|
||||
* nodes deleted)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class DenseLongMap
|
||||
{
|
||||
private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
|
||||
|
||||
private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
|
||||
private int valuebits;
|
||||
private int maxvalue;
|
||||
private long maxkey;
|
||||
private long maxmemory;
|
||||
|
||||
/**
|
||||
* Creates a DenseLongMap for the given value range
|
||||
* Note that one value is reserved for the "unset" state,
|
||||
* so with 6 value bits you can store values in the
|
||||
* range 0..62 only
|
||||
*
|
||||
* @param valuebits number of bits to use per value
|
||||
*/
|
||||
public DenseLongMap( int valuebits )
|
||||
{
|
||||
if ( valuebits < 1 || valuebits > 32 )
|
||||
{
|
||||
throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
|
||||
}
|
||||
this.valuebits = valuebits;
|
||||
maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
|
||||
maxvalue = (1 << valuebits) - 2;
|
||||
maxkey = ( maxmemory / valuebits ) * 8;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void put( long key, int value )
|
||||
{
|
||||
if ( key < 0L || key > maxkey )
|
||||
{
|
||||
throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
|
||||
+ " give more memory (currently " + (maxmemory / 0x100000)
|
||||
+ "MB) to extend key range" );
|
||||
}
|
||||
if ( value < 0 || value > maxvalue )
|
||||
{
|
||||
throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
|
||||
}
|
||||
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
block = new int[BLOCKSIZE * valuebits];
|
||||
|
||||
while (blocklist.size() < blockn+1 )
|
||||
{
|
||||
blocklist.add(null);
|
||||
}
|
||||
blocklist.set( blockn, block );
|
||||
}
|
||||
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int invmask = bitmask ^ 0xffffffff;
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = value + 1; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( v & probebit ) != 0 )
|
||||
{
|
||||
block[blockidx] |= bitmask;
|
||||
}
|
||||
else
|
||||
{
|
||||
block[blockidx] &= invmask;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getInt( long key )
|
||||
{
|
||||
if ( key < 0 )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = 0; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( block[blockidx] & bitmask ) != 0 )
|
||||
{
|
||||
v |= probebit;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
return v-1;
|
||||
}
|
||||
|
||||
}
|
||||
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class DenseLongMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 100000, 100000, 100000 );
|
||||
hashMapComparison( 100000, 100000, 13000000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount, long keyrange )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,Integer> hmap = new HashMap<Long,Integer>();
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
|
||||
hmap.put( KK, new Integer ( value ) );
|
||||
dmap.put( k, value ); // duplicate puts allowed!
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
Integer VV = hmap.get( KK );
|
||||
int hvalue = VV == null ? -1 : VV.intValue();
|
||||
int dvalue = dmap.getInt( k );
|
||||
|
||||
if ( hvalue != dvalue )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashmap=" + hvalue + " densemap=" + dvalue );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void oneBitTest()
|
||||
{
|
||||
int keyrange = 300000;
|
||||
int mapputs = 100000;
|
||||
int trycount = 100000;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
|
||||
DenseLongMap dmap = new DenseLongMap( 1 );
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
hset.add( new Long( k ) );
|
||||
dmap.put( k, 0 );
|
||||
}
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
boolean hcontains = hset.contains( new Long( k ) );
|
||||
boolean dcontains = dmap.getInt( k ) == 0;
|
||||
|
||||
if ( hcontains != dcontains )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashset=" + hcontains + " densemap=" + dcontains );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @Test - memory test disabled for load reasons
|
||||
public void memoryUsageTest()
|
||||
{
|
||||
int keyrange = 32000000;
|
||||
int mapputs = keyrange * 2;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
System.gc();
|
||||
long mem1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
dmap.put( k, value );
|
||||
}
|
||||
|
||||
System.gc();
|
||||
long mem2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
long memusage = mem2-mem1;
|
||||
|
||||
if ( memusage > (keyrange/8)*7 )
|
||||
{
|
||||
Assert.fail( "memory usage too high: " + memusage + " for keyrange " + keyrange );
|
||||
}
|
||||
|
||||
// need to use the map again for valid memory measure
|
||||
Assert.assertTrue( "out of range test", dmap.getInt(-1) == -1 );
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Map
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongMap<V> extends CompactLongMap<V>
|
||||
{
|
||||
private long[] faid;
|
||||
private ArrayList<V> flv;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongMap( CompactLongMap<V> map )
|
||||
{
|
||||
size = map.size();
|
||||
|
||||
faid = new long[size];
|
||||
flv = new ArrayList<V>(size);
|
||||
|
||||
map.moveToFrozenArrays( faid, flv );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = flv.get(n);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the value for "id",
|
||||
* Throw an exception if not contained in the map.
|
||||
*/
|
||||
@Override
|
||||
public V get( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return null;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return flv.get(n);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<V> getValueList()
|
||||
{
|
||||
return flv;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Set
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongSet extends CompactLongSet
|
||||
{
|
||||
private long[] faid;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongSet( CompactLongSet set )
|
||||
{
|
||||
size = set.size();
|
||||
|
||||
faid = new long[size];
|
||||
|
||||
set.moveToFrozenArray( faid );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
public boolean contains( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Behaves like an Array of list
|
||||
* with lazy list-allocation at getList
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LazyArrayOfLists<E>
|
||||
{
|
||||
private ArrayList<ArrayList<E>> lists;
|
||||
|
||||
public LazyArrayOfLists( int size )
|
||||
{
|
||||
lists = new ArrayList<ArrayList<E>>( size );
|
||||
for ( int i = 0; i< size; i++ )
|
||||
{
|
||||
lists.add( null );
|
||||
}
|
||||
}
|
||||
|
||||
public List<E> getList( int idx )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list == null )
|
||||
{
|
||||
list = new ArrayList<E>();
|
||||
lists.set( idx, list );
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public int getSize( int idx )
|
||||
{
|
||||
List<E> list = lists.get( idx );
|
||||
return list == null ? 0 : list.size();
|
||||
}
|
||||
|
||||
public void trimAll()
|
||||
{
|
||||
for ( int idx = 0; idx< lists.size(); idx++ )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list != null )
|
||||
{
|
||||
list.trimToSize();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* dynamic list of primitive longs
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LongList
|
||||
{
|
||||
private long[] a;
|
||||
private int size;
|
||||
|
||||
public LongList( int capacity )
|
||||
{
|
||||
a = capacity < 4 ? new long[4] : new long[capacity];
|
||||
}
|
||||
|
||||
public void add( long value )
|
||||
{
|
||||
if ( size == a.length )
|
||||
{
|
||||
long[] aa = new long[2*size];
|
||||
System.arraycopy( a, 0, aa, 0, size );
|
||||
a = aa;
|
||||
}
|
||||
a[size++] = value;
|
||||
}
|
||||
|
||||
public long get( int idx )
|
||||
{
|
||||
if ( idx >= size )
|
||||
{
|
||||
throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
|
||||
}
|
||||
return a[idx];
|
||||
}
|
||||
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* TinyDenseLongMap implements the DenseLongMap interface
|
||||
* but actually is made for a medium count of non-dense keys
|
||||
*
|
||||
* It's used as a replacement for DenseLongMap where we
|
||||
* have limited memory and far less keys than maykey
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class TinyDenseLongMap extends DenseLongMap
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
|
||||
public TinyDenseLongMap()
|
||||
{
|
||||
super(1);
|
||||
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new byte[MAXLISTS][];
|
||||
vla[0] = new byte[1];
|
||||
}
|
||||
|
||||
|
||||
private byte[][] vla; // value list array
|
||||
|
||||
private void fillReturnValue(byte[] rv, int idx, int p )
|
||||
{
|
||||
rv[0] = vla[idx][p];
|
||||
if ( rv.length == 2 )
|
||||
{
|
||||
vla[idx][p] = rv[1];
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put( long id, int value )
|
||||
{
|
||||
byte[] rv = new byte[2];
|
||||
rv[1] = (byte)value;
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
vla[0][0] = (byte)value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the byte for the given id
|
||||
* @param id the key to query
|
||||
* @return the object
|
||||
* @exception IllegalArgumentException if id is unknown
|
||||
*/
|
||||
@Override
|
||||
public int getInt( long id )
|
||||
{
|
||||
byte[] rv = new byte[1];
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return rv[0];
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new byte[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
private boolean contains( long id, byte[] rv )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, rv ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, byte[] rv )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
if ( rv != null )
|
||||
{
|
||||
fillReturnValue( rv, idx, n );
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue