added crc logic for datafile access
This commit is contained in:
Arndt Brenschede
parent
e88465f5ec
commit
b4aa961f2d
26 changed files with 621 additions and 1364 deletions
67
brouter-util/src/main/java/btools/util/Crc32.java
Normal file
67
brouter-util/src/main/java/btools/util/Crc32.java
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* checksum stuff
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class Crc32
|
||||
{
|
||||
public static int crc( long bitmap )
|
||||
{
|
||||
int crc = 0xFFFFFFFF;
|
||||
long bm = bitmap;
|
||||
for( int j=0; j<8; j++ )
|
||||
{
|
||||
crc = (crc >>> 8) ^ crctable[(crc ^ (int)bm) & 0xff];
|
||||
bm >>= 8;
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
public static int crc( byte[] ab, int offset, int len )
|
||||
{
|
||||
int crc = 0xFFFFFFFF;
|
||||
for( int j=offset; j<len; j++ )
|
||||
{
|
||||
crc = (crc >>> 8) ^ crctable[(crc ^ ab[j]) & 0xff];
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
private static int[] crctable = {
|
||||
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
|
||||
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
|
||||
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
|
||||
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
|
||||
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
|
||||
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
|
||||
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
|
||||
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
|
||||
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
|
||||
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
|
||||
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
|
||||
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
|
||||
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
|
||||
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
|
||||
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
|
||||
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
|
||||
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
|
||||
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
|
||||
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
|
||||
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
|
||||
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
|
||||
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
|
||||
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
|
||||
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
|
||||
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
|
||||
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
|
||||
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
|
||||
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
|
||||
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
|
||||
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
|
||||
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
|
||||
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,328 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Memory efficient Map to map a long-key to an object-value
|
||||
*
|
||||
* Implementation is such that basically the 12 bytes
|
||||
* per entry is allocated that's needed to store
|
||||
* a long- and an object-value.
|
||||
* This class does not implement the Map interface
|
||||
* because it's not complete (remove() is not implemented,
|
||||
* CompactLongMap can only grow.)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongMap<V>
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
private V value_in;
|
||||
protected V value_out;
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongMap()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new Object[MAXLISTS][];
|
||||
vla[0] = new Object[1];
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
*
|
||||
* The Map extension:
|
||||
* next 5 protected methods are needed to implement value-support
|
||||
* overwrite them all to support value structures other than the
|
||||
* long-values implemented here as a sample.
|
||||
*
|
||||
* Furthermore, put() and get() method need to be implemented
|
||||
* to access the values.
|
||||
*
|
||||
* Note that this map does not behave exactly like java.util.Map
|
||||
* - put(..) with already existing key throws exception
|
||||
* - get(..) with non-existing key thros exception
|
||||
*
|
||||
* If you have keys that cannot easily be mapped on long's, use
|
||||
* a hash-function to do the mapping. But note that, in comparison
|
||||
* to java.util.HashMap, in that case the keys itself are not saved,
|
||||
* only the hash-values, so you need to be sure that random duplicate
|
||||
* hashs are either excluded by the structure of your data or that
|
||||
* you can handle the possible IllegalArgumentException
|
||||
*
|
||||
*/
|
||||
|
||||
private Object[][] vla; // value list array
|
||||
|
||||
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
try
|
||||
{
|
||||
value_in = value;
|
||||
if ( contains( id, true ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_in = null;
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as put( id, value ) but duplicate check
|
||||
* is skipped for performance. Be aware that you
|
||||
* can get a duplicate exception later on if the
|
||||
* map is restructured!
|
||||
* with System parameter earlyDuplicateCheck=true you
|
||||
* can enforce the early duplicate check for debugging
|
||||
*
|
||||
* @param id the key to insert
|
||||
* @param value the value to insert object
|
||||
* @exception IllegalArgumentException for duplicates if enabled
|
||||
*/
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
if ( earlyDuplicateCheck && contains( id ) )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in early check: " + id );
|
||||
}
|
||||
vla[0][0] = value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value for the given id
|
||||
* @param id the key to query
|
||||
* @return the object, or null if id not known
|
||||
*/
|
||||
public V get( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
if ( contains( id, false ) )
|
||||
{
|
||||
return value_out;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this map
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new Object[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
try
|
||||
{
|
||||
return contains( id, false );
|
||||
}
|
||||
finally
|
||||
{
|
||||
value_out = null;
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, doPut ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, boolean doPut )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = (V)vla[idx][n];
|
||||
if ( doPut ) vla[idx][n] = value_in;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArrays( long[] faid, ArrayList<V> flv )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
flv.add( (V)vla[minIdx][pa[minIdx]] );
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen arrays
|
||||
al = null;
|
||||
vla = null;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Memory efficient Set for long-keys
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class CompactLongSet
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
private static boolean earlyDuplicateCheck;
|
||||
|
||||
public CompactLongSet()
|
||||
{
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
/**
|
||||
* add a long value to this set if not yet in.
|
||||
* @param id the value to add to this set.
|
||||
* @return true if "id" already contained in this set.
|
||||
*/
|
||||
public boolean add( long id )
|
||||
{
|
||||
if ( contains( id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_add( id );
|
||||
return false;
|
||||
}
|
||||
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
_add( id );
|
||||
}
|
||||
|
||||
private void _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
public boolean contains( long id )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void moveToFrozenArray( long[] faid )
|
||||
{
|
||||
for( int i=1; i<MAXLISTS; i++ )
|
||||
{
|
||||
pa[i] = 0;
|
||||
}
|
||||
|
||||
for( int ti = 0; ti < size; ti++ ) // target-index
|
||||
{
|
||||
int bp = size; // treat size as bitpattern
|
||||
int minIdx = -1;
|
||||
long minId = 0;
|
||||
int idx = 1;
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
int p = pa[idx];
|
||||
if ( p < al[idx].length )
|
||||
{
|
||||
long currentId = al[idx][p];
|
||||
if ( minIdx < 0 || currentId < minId )
|
||||
{
|
||||
minIdx = idx;
|
||||
minId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
faid[ti] = minId;
|
||||
pa[minIdx]++;
|
||||
|
||||
if ( ti > 0 && faid[ti-1] == minId )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
|
||||
}
|
||||
}
|
||||
|
||||
// free the non-frozen array
|
||||
al = null;
|
||||
}
|
||||
|
||||
}
|
||||
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
68
brouter-util/src/test/java/btools/util/CompactMapTest.java
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 0, 1 );
|
||||
hashMapComparison( 1, 1 );
|
||||
hashMapComparison( 2, 2 );
|
||||
hashMapComparison( 3, 3 );
|
||||
hashMapComparison( 4, 4 );
|
||||
hashMapComparison( 5, 5 );
|
||||
hashMapComparison( 7, 10 );
|
||||
hashMapComparison( 8, 10 );
|
||||
hashMapComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,String> hmap = new HashMap<Long,String>();
|
||||
CompactLongMap<String> cmap_slow = new CompactLongMap<String>();
|
||||
CompactLongMap<String> cmap_fast = new CompactLongMap<String>();
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
String s = "" + i;
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hmap.containsKey( KK ) )
|
||||
{
|
||||
hmap.put( KK, s );
|
||||
cmap_slow.put( k, s );
|
||||
cmap_fast.fastPut( k, s );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cmap_slow = new FrozenLongMap<String>( cmap_slow );
|
||||
cmap_fast = new FrozenLongMap<String>( cmap_fast );
|
||||
}
|
||||
long k = mapsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
String s = hmap.get( KK );
|
||||
|
||||
boolean contained = hmap.containsKey( KK );
|
||||
Assert.assertTrue( "containsKey missmatch (slow)", contained == cmap_slow.contains( k ) );
|
||||
Assert.assertTrue( "containsKey missmatch (fast)", contained == cmap_fast.contains( k ) );
|
||||
|
||||
if ( contained )
|
||||
{
|
||||
Assert.assertEquals( "object missmatch (fast)", s, cmap_fast.get( k ) );
|
||||
Assert.assertEquals( "object missmatch (slow)", s, cmap_slow.get( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
60
brouter-util/src/test/java/btools/util/CompactSetTest.java
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class CompactSetTest
|
||||
{
|
||||
@Test
|
||||
public void hashSetComparisonTest()
|
||||
{
|
||||
hashSetComparison( 0, 1 );
|
||||
hashSetComparison( 1, 1 );
|
||||
hashSetComparison( 2, 2 );
|
||||
hashSetComparison( 3, 3 );
|
||||
hashSetComparison( 4, 4 );
|
||||
hashSetComparison( 5, 5 );
|
||||
hashSetComparison( 7, 10 );
|
||||
hashSetComparison( 8, 10 );
|
||||
hashSetComparison( 10000, 20000 );
|
||||
}
|
||||
|
||||
private void hashSetComparison( int setsize, int trycount )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
CompactLongSet cset_slow = new CompactLongSet();
|
||||
CompactLongSet cset_fast = new CompactLongSet();
|
||||
|
||||
for( int i=0; i<setsize; i++ )
|
||||
{
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
if ( !hset.contains( KK ) )
|
||||
{
|
||||
hset.add( KK );
|
||||
cset_slow.add( k );
|
||||
cset_fast.fastAdd( k );
|
||||
}
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount*2; i++ )
|
||||
{
|
||||
if ( i == trycount )
|
||||
{
|
||||
cset_slow = new FrozenLongSet( cset_slow );
|
||||
cset_fast = new FrozenLongSet( cset_fast );
|
||||
}
|
||||
long k = setsize < 10 ? i : rand.nextInt( 20000 );
|
||||
Long KK = new Long( k );
|
||||
|
||||
boolean contained = hset.contains( KK );
|
||||
Assert.assertTrue( "contains missmatch (slow)", contained == cset_slow.contains( k ) );
|
||||
Assert.assertTrue( "contains missmatch (fast)", contained == cset_fast.contains( k ) );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Special Memory efficient Map to map a long-key to
|
||||
* a "small" value (some bits only) where it is expected
|
||||
* that the keys are dense, so that we can use more or less
|
||||
* a simple array as the best-fit data model (except for
|
||||
* the 32-bit limit of arrays!)
|
||||
*
|
||||
* Additionally, to enable small-memory unit testing
|
||||
* of code using this map this one has a fallback for a small map
|
||||
* where we have only few keys, but not dense. In this
|
||||
* case, we use the mechanics of the CompactLongMap
|
||||
*
|
||||
* Target application are osm-node ids which are in the
|
||||
* range 0...3 billion and basically dense (=only few
|
||||
* nodes deleted)
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class DenseLongMap
|
||||
{
|
||||
private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
|
||||
|
||||
private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
|
||||
private int valuebits;
|
||||
private int maxvalue;
|
||||
private long maxkey;
|
||||
private long maxmemory;
|
||||
|
||||
/**
|
||||
* Creates a DenseLongMap for the given value range
|
||||
* Note that one value is reserved for the "unset" state,
|
||||
* so with 6 value bits you can store values in the
|
||||
* range 0..62 only
|
||||
*
|
||||
* @param valuebits number of bits to use per value
|
||||
*/
|
||||
public DenseLongMap( int valuebits )
|
||||
{
|
||||
if ( valuebits < 1 || valuebits > 32 )
|
||||
{
|
||||
throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
|
||||
}
|
||||
this.valuebits = valuebits;
|
||||
maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
|
||||
maxvalue = (1 << valuebits) - 2;
|
||||
maxkey = ( maxmemory / valuebits ) * 8;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void put( long key, int value )
|
||||
{
|
||||
if ( key < 0L || key > maxkey )
|
||||
{
|
||||
throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
|
||||
+ " give more memory (currently " + (maxmemory / 0x100000)
|
||||
+ "MB) to extend key range" );
|
||||
}
|
||||
if ( value < 0 || value > maxvalue )
|
||||
{
|
||||
throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
|
||||
}
|
||||
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
block = new int[BLOCKSIZE * valuebits];
|
||||
|
||||
while (blocklist.size() < blockn+1 )
|
||||
{
|
||||
blocklist.add(null);
|
||||
}
|
||||
blocklist.set( blockn, block );
|
||||
}
|
||||
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int invmask = bitmask ^ 0xffffffff;
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = value + 1; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( v & probebit ) != 0 )
|
||||
{
|
||||
block[blockidx] |= bitmask;
|
||||
}
|
||||
else
|
||||
{
|
||||
block[blockidx] &= invmask;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public int getInt( long key )
|
||||
{
|
||||
if ( key < 0 )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int blockn = (int)(key >> 21);
|
||||
int offset = (int)(key & 0x1fffff);
|
||||
|
||||
int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
|
||||
|
||||
if ( block == null )
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
int bitmask = 1 << (offset & 0x1f);
|
||||
int probebit = 1;
|
||||
int blockidx = (offset >> 5)*valuebits;
|
||||
int blockend = blockidx + valuebits;
|
||||
int v = 0; // 0 is reserved (=unset)
|
||||
|
||||
while( blockidx < blockend )
|
||||
{
|
||||
if ( ( block[blockidx] & bitmask ) != 0 )
|
||||
{
|
||||
v |= probebit;
|
||||
}
|
||||
probebit <<= 1;
|
||||
blockidx++;
|
||||
}
|
||||
return v-1;
|
||||
}
|
||||
|
||||
}
|
||||
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
113
brouter-util/src/test/java/btools/util/DenseLongMapTest.java
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class DenseLongMapTest
|
||||
{
|
||||
@Test
|
||||
public void hashMapComparisonTest()
|
||||
{
|
||||
hashMapComparison( 100000, 100000, 100000 );
|
||||
hashMapComparison( 100000, 100000, 13000000 );
|
||||
}
|
||||
|
||||
private void hashMapComparison( int mapsize, int trycount, long keyrange )
|
||||
{
|
||||
Random rand = new Random( 12345 );
|
||||
HashMap<Long,Integer> hmap = new HashMap<Long,Integer>();
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
for( int i=0; i<mapsize; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
|
||||
hmap.put( KK, new Integer ( value ) );
|
||||
dmap.put( k, value ); // duplicate puts allowed!
|
||||
}
|
||||
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
Long KK = new Long( k );
|
||||
Integer VV = hmap.get( KK );
|
||||
int hvalue = VV == null ? -1 : VV.intValue();
|
||||
int dvalue = dmap.getInt( k );
|
||||
|
||||
if ( hvalue != dvalue )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashmap=" + hvalue + " densemap=" + dvalue );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void oneBitTest()
|
||||
{
|
||||
int keyrange = 300000;
|
||||
int mapputs = 100000;
|
||||
int trycount = 100000;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
HashSet<Long> hset = new HashSet<Long>();
|
||||
|
||||
DenseLongMap dmap = new DenseLongMap( 1 );
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
hset.add( new Long( k ) );
|
||||
dmap.put( k, 0 );
|
||||
}
|
||||
for( int i=0; i<trycount; i++ )
|
||||
{
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
boolean hcontains = hset.contains( new Long( k ) );
|
||||
boolean dcontains = dmap.getInt( k ) == 0;
|
||||
|
||||
if ( hcontains != dcontains )
|
||||
{
|
||||
Assert.fail( "value missmatch for key " + k + " hashset=" + hcontains + " densemap=" + dcontains );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @Test - memory test disabled for load reasons
|
||||
public void memoryUsageTest()
|
||||
{
|
||||
int keyrange = 32000000;
|
||||
int mapputs = keyrange * 2;
|
||||
|
||||
Random rand = new Random( 12345 );
|
||||
DenseLongMap dmap = new DenseLongMap( 6 );
|
||||
|
||||
System.gc();
|
||||
long mem1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
for( int i=0; i<mapputs; i++ )
|
||||
{
|
||||
int value = i%63;
|
||||
long k = (long)(rand.nextDouble()*keyrange);
|
||||
dmap.put( k, value );
|
||||
}
|
||||
|
||||
System.gc();
|
||||
long mem2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
|
||||
long memusage = mem2-mem1;
|
||||
|
||||
if ( memusage > (keyrange/8)*7 )
|
||||
{
|
||||
Assert.fail( "memory usage too high: " + memusage + " for keyrange " + keyrange );
|
||||
}
|
||||
|
||||
// need to use the map again for valid memory measure
|
||||
Assert.assertTrue( "out of range test", dmap.getInt(-1) == -1 );
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Map
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongMap<V> extends CompactLongMap<V>
|
||||
{
|
||||
private long[] faid;
|
||||
private ArrayList<V> flv;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongMap( CompactLongMap<V> map )
|
||||
{
|
||||
size = map.size();
|
||||
|
||||
faid = new long[size];
|
||||
flv = new ArrayList<V>(size);
|
||||
|
||||
map.moveToFrozenArrays( faid, flv );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean put( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastPut( long id, V value )
|
||||
{
|
||||
throw new RuntimeException( "cannot put on FrozenLongIntMap" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
protected boolean contains( long id, boolean doPut )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
value_out = flv.get(n);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the value for "id",
|
||||
* Throw an exception if not contained in the map.
|
||||
*/
|
||||
@Override
|
||||
public V get( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return null;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return flv.get(n);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<V> getValueList()
|
||||
{
|
||||
return flv;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* Frozen instance of Memory efficient Set
|
||||
*
|
||||
* This one is readily sorted into a singe array for faster access
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class FrozenLongSet extends CompactLongSet
|
||||
{
|
||||
private long[] faid;
|
||||
private int size = 0;
|
||||
private int p2size; // next power of 2 of size
|
||||
|
||||
public FrozenLongSet( CompactLongSet set )
|
||||
{
|
||||
size = set.size();
|
||||
|
||||
faid = new long[size];
|
||||
|
||||
set.moveToFrozenArray( faid );
|
||||
|
||||
p2size = 0x40000000;
|
||||
while( p2size > size ) p2size >>= 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean add( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fastAdd( long id )
|
||||
{
|
||||
throw new RuntimeException( "cannot add on FrozenLongSet" );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of entries in this set
|
||||
*/
|
||||
@Override
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @return true if "id" is contained in this set.
|
||||
*/
|
||||
@Override
|
||||
public boolean contains( long id )
|
||||
{
|
||||
if ( size == 0 )
|
||||
{
|
||||
return false;
|
||||
}
|
||||
long[] a = faid;
|
||||
int offset = p2size;
|
||||
int n = 0;
|
||||
|
||||
while ( offset> 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( nn < size && a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
offset >>= 1;
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
|
||||
/**
|
||||
* Behaves like an Array of list
|
||||
* with lazy list-allocation at getList
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LazyArrayOfLists<E>
|
||||
{
|
||||
private ArrayList<ArrayList<E>> lists;
|
||||
|
||||
public LazyArrayOfLists( int size )
|
||||
{
|
||||
lists = new ArrayList<ArrayList<E>>( size );
|
||||
for ( int i = 0; i< size; i++ )
|
||||
{
|
||||
lists.add( null );
|
||||
}
|
||||
}
|
||||
|
||||
public List<E> getList( int idx )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list == null )
|
||||
{
|
||||
list = new ArrayList<E>();
|
||||
lists.set( idx, list );
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public int getSize( int idx )
|
||||
{
|
||||
List<E> list = lists.get( idx );
|
||||
return list == null ? 0 : list.size();
|
||||
}
|
||||
|
||||
public void trimAll()
|
||||
{
|
||||
for ( int idx = 0; idx< lists.size(); idx++ )
|
||||
{
|
||||
ArrayList<E> list = lists.get( idx );
|
||||
if ( list != null )
|
||||
{
|
||||
list.trimToSize();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* dynamic list of primitive longs
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class LongList
|
||||
{
|
||||
private long[] a;
|
||||
private int size;
|
||||
|
||||
public LongList( int capacity )
|
||||
{
|
||||
a = capacity < 4 ? new long[4] : new long[capacity];
|
||||
}
|
||||
|
||||
public void add( long value )
|
||||
{
|
||||
if ( size == a.length )
|
||||
{
|
||||
long[] aa = new long[2*size];
|
||||
System.arraycopy( a, 0, aa, 0, size );
|
||||
a = aa;
|
||||
}
|
||||
a[size++] = value;
|
||||
}
|
||||
|
||||
public long get( int idx )
|
||||
{
|
||||
if ( idx >= size )
|
||||
{
|
||||
throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
|
||||
}
|
||||
return a[idx];
|
||||
}
|
||||
|
||||
public int size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,206 +0,0 @@
|
|||
package btools.util;
|
||||
|
||||
/**
|
||||
* TinyDenseLongMap implements the DenseLongMap interface
|
||||
* but actually is made for a medium count of non-dense keys
|
||||
*
|
||||
* It's used as a replacement for DenseLongMap where we
|
||||
* have limited memory and far less keys than maykey
|
||||
*
|
||||
* @author ab
|
||||
*/
|
||||
public class TinyDenseLongMap extends DenseLongMap
|
||||
{
|
||||
private long[][] al;
|
||||
private int[] pa;
|
||||
private int size = 0;
|
||||
private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
|
||||
|
||||
protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
|
||||
|
||||
public TinyDenseLongMap()
|
||||
{
|
||||
super(1);
|
||||
|
||||
// pointer array
|
||||
pa = new int[MAXLISTS];
|
||||
|
||||
// allocate key lists
|
||||
al = new long[MAXLISTS][];
|
||||
al[0] = new long[1]; // make the first array (the transient buffer)
|
||||
|
||||
// same for the values
|
||||
vla = new byte[MAXLISTS][];
|
||||
vla[0] = new byte[1];
|
||||
}
|
||||
|
||||
|
||||
private byte[][] vla; // value list array
|
||||
|
||||
private void fillReturnValue(byte[] rv, int idx, int p )
|
||||
{
|
||||
rv[0] = vla[idx][p];
|
||||
if ( rv.length == 2 )
|
||||
{
|
||||
vla[idx][p] = rv[1];
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put( long id, int value )
|
||||
{
|
||||
byte[] rv = new byte[2];
|
||||
rv[1] = (byte)value;
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
vla[0][0] = (byte)value;
|
||||
_add( id );
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get the byte for the given id
|
||||
* @param id the key to query
|
||||
* @return the object
|
||||
* @exception IllegalArgumentException if id is unknown
|
||||
*/
|
||||
@Override
|
||||
public int getInt( long id )
|
||||
{
|
||||
byte[] rv = new byte[1];
|
||||
if ( contains( id, rv ) )
|
||||
{
|
||||
return rv[0];
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
private boolean _add( long id )
|
||||
{
|
||||
if ( size == Integer.MAX_VALUE )
|
||||
{
|
||||
throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
|
||||
}
|
||||
|
||||
// put the new entry in the first array
|
||||
al[0][0] = id;
|
||||
|
||||
// determine the first empty array
|
||||
int bp = size++; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
int n = 1;
|
||||
|
||||
pa[0] = 1;
|
||||
pa[1] = 1;
|
||||
|
||||
while ( (bp&1) == 1 )
|
||||
{
|
||||
bp >>= 1;
|
||||
pa[idx++] = n;
|
||||
n <<= 1;
|
||||
}
|
||||
|
||||
// create it if not existant
|
||||
if ( al[idx] == null )
|
||||
{
|
||||
al[idx] = new long[n];
|
||||
vla[idx] = new byte[n];
|
||||
}
|
||||
|
||||
// now merge the contents of arrays 0...idx-1 into idx
|
||||
while ( n > 0 )
|
||||
{
|
||||
long maxId = 0;
|
||||
int maxIdx = -1;
|
||||
|
||||
for ( int i=0; i<idx; i++ )
|
||||
{
|
||||
int p = pa[i];
|
||||
if ( p > 0 )
|
||||
{
|
||||
long currentId = al[i][p-1];
|
||||
if ( maxIdx < 0 || currentId > maxId )
|
||||
{
|
||||
maxIdx = i;
|
||||
maxId = currentId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// current maximum found, copy to target array
|
||||
if ( n < al[idx].length && maxId == al[idx][n] )
|
||||
{
|
||||
throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
|
||||
}
|
||||
--n;
|
||||
al[idx][n] = maxId;
|
||||
vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
|
||||
|
||||
--pa[maxIdx];
|
||||
}
|
||||
|
||||
// de-allocate empty arrays of a certain size (fix at 64kByte)
|
||||
while ( idx-- > _maxKeepExponent )
|
||||
{
|
||||
al[idx] = null;
|
||||
vla[idx] = null;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
private boolean contains( long id, byte[] rv )
|
||||
{
|
||||
// determine the first empty array
|
||||
int bp = size; // treat size as bitpattern
|
||||
int idx = 1;
|
||||
|
||||
while ( bp != 0 )
|
||||
{
|
||||
if ( (bp&1) == 1 )
|
||||
{
|
||||
// array at idx is valid, check
|
||||
if ( contains( idx, id, rv ) )
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
idx++;
|
||||
bp >>= 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// does sorted array "a" contain "id" ?
|
||||
private boolean contains( int idx, long id, byte[] rv )
|
||||
{
|
||||
long[] a = al[idx];
|
||||
int offset = a.length;
|
||||
int n = 0;
|
||||
|
||||
while ( (offset >>= 1) > 0 )
|
||||
{
|
||||
int nn = n + offset;
|
||||
if ( a[nn] <= id )
|
||||
{
|
||||
n = nn;
|
||||
}
|
||||
}
|
||||
if ( a[n] == id )
|
||||
{
|
||||
if ( rv != null )
|
||||
{
|
||||
fillReturnValue( rv, idx, n );
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue