initial commit of BRouter Version 0.98

brouter-util/pom.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.btools</groupId>
+        <artifactId>brouter</artifactId>
+        <version>0.98</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>brouter-util</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+      <dependency>
+        <groupId>junit</groupId>
+			  <artifactId>junit</artifactId>
+		  </dependency>
+	  </dependencies>
+
+</project>

brouter-util/src/main/java/btools/util/CompactLongMap.java

@@ -0,0 +1,328 @@
+package btools.util;
+
+import java.util.ArrayList;
+
+/**
+ * Memory efficient Map to map a long-key to an object-value
+ *
+ * Implementation is such that basically the 12 bytes
+ * per entry is allocated that's needed to store
+ * a long- and an object-value.
+ * This class does not implement the Map interface
+ * because it's not complete (remove() is not implemented,
+ * CompactLongMap can only grow.)
+ *
+ * @author ab
+ */
+public class CompactLongMap<V>
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  private V value_in;
+  protected V value_out;
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+  private static boolean earlyDuplicateCheck;
+
+  public CompactLongMap()
+  {
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    // same for the values
+    vla = new Object[MAXLISTS][];
+    vla[0] = new Object[1];
+
+    earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
+  }
+
+
+  /*
+   *
+   * The Map extension:
+   * next 5 protected methods are needed to implement value-support
+   * overwrite them all to support value structures other than the
+   * long-values implemented here as a sample.
+   *
+   * Furthermore, put() and get() method need to be implemented
+   * to access the values.
+   *
+   * Note that this map does not behave exactly like java.util.Map
+   * - put(..) with already existing key throws exception
+   * - get(..) with non-existing key thros exception
+   *
+   * If you have keys that cannot easily be mapped on long's, use
+   * a hash-function to do the mapping. But note that, in comparison
+   * to java.util.HashMap, in that case the keys itself are not saved,
+   * only the hash-values, so you need to be sure that random duplicate
+   * hashs are either excluded by the structure of your data or that
+   * you can handle the possible IllegalArgumentException
+   *
+   */
+
+  private Object[][] vla; // value list array
+
+
+  public boolean put( long id, V value )
+  {
+    try
+    {    
+      value_in = value;
+      if ( contains( id, true ) )
+      {
+        return true;
+      }
+      vla[0][0] = value;
+      _add( id );
+      return false;
+    }
+    finally
+    {
+      value_in = null;
+      value_out = null;
+    }
+  }
+
+  /**
+   * Same as put( id, value ) but duplicate check
+   * is skipped for performance. Be aware that you
+   * can get a duplicate exception later on if the
+   * map is restructured!
+   * with System parameter earlyDuplicateCheck=true you
+   * can enforce the early duplicate check for debugging
+   *
+   * @param id the key to insert
+   * @param value the value to insert object
+   * @exception IllegalArgumentException for duplicates if enabled
+   */
+  public void fastPut( long id, V value )
+  {
+    if ( earlyDuplicateCheck && contains( id ) )
+    {
+      throw new IllegalArgumentException( "duplicate key found in early check: " + id );
+    }
+    vla[0][0] = value;
+    _add( id );
+  }
+
+  /**
+   * Get the value for the given id
+   * @param id the key to query
+   * @return the object, or null if id not known
+   */
+  public V get( long id )
+  {
+    try
+    {
+      if ( contains( id, false ) )
+      {
+        return value_out;
+      }
+      return null;
+    }
+    finally
+    {
+      value_out = null;
+    }
+  }
+
+
+  /**
+   * @return the number of entries in this map
+   */
+  public int size()
+  {
+    return size;
+  }
+
+
+  private boolean _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+      vla[idx] = new Object[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+      vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+      vla[idx] = null;
+    }
+
+    return false;
+  }
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  public boolean contains( long id )
+  {
+    try
+    {
+      return contains( id, false );
+    }
+    finally
+    {
+      value_out = null;
+    }
+  }
+
+  protected boolean contains( long id, boolean doPut )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id, doPut ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id, boolean doPut )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      value_out = (V)vla[idx][n];
+      if ( doPut ) vla[idx][n] = value_in;
+      return true;
+    }
+    return false;
+  }
+
+  protected void moveToFrozenArrays( long[] faid, ArrayList<V> flv )
+  {
+    for( int i=1; i<MAXLISTS; i++ )
+    {
+      pa[i] = 0;
+    }
+
+    for( int ti = 0; ti < size; ti++ ) // target-index
+    {
+      int bp = size; // treat size as bitpattern
+      int minIdx = -1;
+      long minId = 0;
+      int idx = 1;
+      while ( bp != 0 )
+      {
+        if ( (bp&1) == 1 )
+        {
+          int p = pa[idx];
+          if ( p < al[idx].length )
+          {
+            long currentId = al[idx][p];
+            if ( minIdx < 0 || currentId < minId )
+            {
+              minIdx = idx;
+              minId = currentId;
+            }
+          }
+        }
+        idx++;
+        bp >>= 1;
+      }
+      faid[ti] = minId;
+      flv.add( (V)vla[minIdx][pa[minIdx]] );
+      pa[minIdx]++;
+
+      if ( ti > 0 && faid[ti-1] == minId )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
+      }
+    }
+
+    // free the non-frozen arrays
+    al = null;
+    vla = null;
+  }
+
+}

brouter-util/src/main/java/btools/util/CompactLongSet.java

@@ -0,0 +1,220 @@
+package btools.util;
+
+/**
+ * Memory efficient Set for long-keys
+ *
+ * @author ab
+ */
+public class CompactLongSet
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+  private static boolean earlyDuplicateCheck;
+
+  public CompactLongSet()
+  {
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
+  }
+
+
+  /**
+   * @return the number of entries in this set
+   */
+  public int size()
+  {
+    return size;
+  }
+
+  /**
+   * add a long value to this set if not yet in.
+   * @param id the value to add to this set.
+   * @return true if "id" already contained in this set.
+   */
+  public boolean add( long id )
+  {
+    if ( contains( id ) )
+    {
+      return true;
+    }
+    _add( id );
+    return false;
+  }
+
+  public void fastAdd( long id )
+  {
+    _add( id );
+  }
+
+  private void _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+    }
+  }
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  public boolean contains( long id )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      return true;
+    }
+    return false;
+  }
+
+  protected void moveToFrozenArray( long[] faid )
+  {
+    for( int i=1; i<MAXLISTS; i++ )
+    {
+      pa[i] = 0;
+    }
+
+    for( int ti = 0; ti < size; ti++ ) // target-index
+    {
+      int bp = size; // treat size as bitpattern
+      int minIdx = -1;
+      long minId = 0;
+      int idx = 1;
+      while ( bp != 0 )
+      {
+        if ( (bp&1) == 1 )
+        {
+          int p = pa[idx];
+          if ( p < al[idx].length )
+          {
+            long currentId = al[idx][p];
+            if ( minIdx < 0 || currentId < minId )
+            {
+              minIdx = idx;
+              minId = currentId;
+            }
+          }
+        }
+        idx++;
+        bp >>= 1;
+      }
+      faid[ti] = minId;
+      pa[minIdx]++;
+
+      if ( ti > 0 && faid[ti-1] == minId )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
+      }
+    }
+
+    // free the non-frozen array
+    al = null;
+  }
+
+}

brouter-util/src/main/java/btools/util/DenseLongMap.java

@@ -0,0 +1,140 @@
+package btools.util;
+
+import java.util.ArrayList;
+
+/**
+ * Special Memory efficient Map to map a long-key to
+ * a "small" value (some bits only) where it is expected
+ * that the keys are dense, so that we can use more or less
+ * a simple array as the best-fit data model (except for
+ * the 32-bit limit of arrays!)
+ *
+ * Additionally, to enable small-memory unit testing 
+ * of code using this map this one has a fallback for a small map
+ * where we have only few keys, but not dense. In this
+ * case, we use the mechanics of the CompactLongMap
+ *
+ * Target application are osm-node ids which are in the
+ * range 0...3 billion and basically dense (=only few
+ * nodes deleted)
+ *
+ * @author ab
+ */
+public class DenseLongMap
+{
+  private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
+
+  private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
+  private int valuebits;
+  private int maxvalue;
+  private long maxkey;
+  private long maxmemory;
+
+  /**
+   * Creates a DenseLongMap for the given value range
+   * Note that one value is reserved for the "unset" state,
+   * so with 6 value bits you can store values in the
+   * range 0..62 only
+   *
+   * @param valuebits number of bits to use per value
+   */
+  public DenseLongMap( int valuebits )
+  {
+    if ( valuebits < 1 || valuebits > 32 )
+    {
+      throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
+    }
+    this.valuebits = valuebits;
+    maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
+    maxvalue = (1 << valuebits) - 2;
+    maxkey = ( maxmemory / valuebits ) * 8;
+  }
+
+
+
+  public void put( long key, int value )
+  {
+    if ( key < 0L || key > maxkey )
+    {
+      throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
+            + " give more memory (currently " + (maxmemory / 0x100000)
+            + "MB) to extend key range" );
+    }
+    if ( value < 0 || value > maxvalue )
+    {
+      throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
+    }
+
+    int blockn = (int)(key >> 21);
+    int offset = (int)(key & 0x1fffff);
+
+    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+
+    if ( block == null )
+    {
+      block = new int[BLOCKSIZE * valuebits];
+
+      while (blocklist.size() < blockn+1 )
+      {
+        blocklist.add(null);
+      }      
+      blocklist.set( blockn, block );
+    }
+    
+    int bitmask = 1 << (offset & 0x1f);
+    int invmask = bitmask ^ 0xffffffff;
+    int probebit = 1;
+    int blockidx = (offset >> 5)*valuebits;
+    int blockend = blockidx + valuebits;
+    int v = value + 1; // 0 is reserved (=unset)
+
+    while( blockidx < blockend )
+    {
+      if ( ( v & probebit ) != 0 )
+      {
+        block[blockidx] |= bitmask;
+      }
+      else
+      {
+        block[blockidx] &= invmask;
+      }
+      probebit <<= 1;
+      blockidx++;
+    }
+  }
+
+
+  public int getInt( long key )
+  {
+    if ( key < 0 )
+    {
+      return -1;
+    }
+    int blockn = (int)(key >> 21);
+    int offset = (int)(key & 0x1fffff);
+
+    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+
+    if ( block == null )
+    {
+      return -1;
+    }
+    int bitmask = 1 << (offset & 0x1f);
+    int probebit = 1;
+    int blockidx = (offset >> 5)*valuebits;
+    int blockend = blockidx + valuebits;
+    int v = 0; // 0 is reserved (=unset)
+
+    while( blockidx < blockend )
+    {
+      if ( ( block[blockidx] & bitmask ) != 0 )
+      {
+        v  |= probebit;
+      }
+      probebit <<= 1;
+      blockidx++;
+    }
+    return v-1;
+  }
+
+}

brouter-util/src/main/java/btools/util/FrozenLongMap.java

@@ -0,0 +1,122 @@
+package btools.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Frozen instance of Memory efficient Map
+ *
+ * This one is readily sorted into a singe array for faster access
+ *
+ * @author ab
+ */
+public class FrozenLongMap<V> extends CompactLongMap<V>
+{
+  private long[] faid;
+  private ArrayList<V> flv;
+  private int size = 0;
+  private int p2size; // next power of 2 of size
+
+  public FrozenLongMap( CompactLongMap<V> map )
+  {
+    size = map.size();
+
+    faid = new long[size];
+    flv = new ArrayList<V>(size);
+
+    map.moveToFrozenArrays( faid, flv );
+
+    p2size = 0x40000000;
+    while( p2size > size ) p2size >>= 1;
+  }
+
+  @Override
+  public boolean put( long id, V value )
+  {
+    throw new RuntimeException( "cannot put on FrozenLongIntMap" );
+  }
+
+  @Override
+  public void fastPut( long id, V value )
+  {
+    throw new RuntimeException( "cannot put on FrozenLongIntMap" );
+  }
+
+  /**
+   * @return the number of entries in this set
+   */
+  @Override
+  public int size()
+  {
+    return size;
+  }
+
+
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  @Override
+  protected boolean contains( long id, boolean doPut )
+  {
+    if ( size == 0 )
+    {
+      return false;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      value_out = flv.get(n);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * @return the value for "id",
+   * Throw an exception if not contained in the map.
+   */
+  @Override
+  public V get( long id )
+  {
+    if ( size == 0 )
+    {
+      return null;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      return flv.get(n);
+    }
+    return null;
+  }
+
+  public List<V> getValueList()
+  {
+    return flv;
+  }
+}

brouter-util/src/main/java/btools/util/FrozenLongSet.java

@@ -0,0 +1,81 @@
+package btools.util;
+
+/**
+ * Frozen instance of Memory efficient Set
+ *
+ * This one is readily sorted into a singe array for faster access
+ *
+ * @author ab
+ */
+public class FrozenLongSet extends CompactLongSet
+{
+  private long[] faid;
+  private int size = 0;
+  private int p2size; // next power of 2 of size
+
+  public FrozenLongSet( CompactLongSet set )
+  {
+    size = set.size();
+
+    faid = new long[size];
+
+    set.moveToFrozenArray( faid );
+
+    p2size = 0x40000000;
+    while( p2size > size ) p2size >>= 1;
+  }
+
+  @Override
+  public boolean add( long id )
+  {
+    throw new RuntimeException( "cannot add on FrozenLongSet" );
+  }
+
+  @Override
+  public void fastAdd( long id )
+  {
+    throw new RuntimeException( "cannot add on FrozenLongSet" );
+  }
+
+  /**
+   * @return the number of entries in this set
+   */
+  @Override
+  public int size()
+  {
+    return size;
+  }
+
+
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  @Override
+  public boolean contains( long id )
+  {
+    if ( size == 0 )
+    {
+      return false;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      return true;
+    }
+    return false;
+  }
+
+}

brouter-util/src/main/java/btools/util/LazyArrayOfLists.java

@@ -0,0 +1,53 @@
+package btools.util;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ * Behaves like an Array of list
+ * with lazy list-allocation at getList
+ *
+ * @author ab
+ */
+public class LazyArrayOfLists<E>
+{
+  private ArrayList<ArrayList<E>> lists;
+
+  public LazyArrayOfLists( int size )
+  {
+    lists = new ArrayList<ArrayList<E>>( size );
+    for ( int i = 0; i< size; i++ )
+    {
+      lists.add( null );
+    }
+  }
+
+  public List<E> getList( int idx )
+  {
+    ArrayList<E> list = lists.get( idx );
+    if ( list == null )
+    {
+      list = new ArrayList<E>();
+      lists.set( idx, list );
+    }
+    return list;
+  }
+
+  public int getSize( int idx )
+  {
+    List<E> list = lists.get( idx );
+    return list == null ? 0 : list.size();
+  }
+    
+  public void trimAll()
+  {
+    for ( int idx = 0; idx< lists.size(); idx++ )
+    {
+      ArrayList<E> list = lists.get( idx );
+      if ( list != null )
+      {
+        list.trimToSize();
+      }
+    }
+  }
+}

brouter-util/src/main/java/btools/util/LongList.java

@@ -0,0 +1,43 @@
+package btools.util;
+
+/**
+ * dynamic list of primitive longs
+ *
+ * @author ab
+ */
+public class LongList
+{
+  private long[] a;
+  private int size;
+
+  public LongList( int capacity )
+  {
+    a = capacity < 4 ? new long[4] : new long[capacity];
+  }
+
+  public void add( long value )
+  {
+    if ( size == a.length )
+    {
+      long[] aa = new long[2*size];
+      System.arraycopy( a, 0, aa, 0, size );
+      a = aa;
+    }
+    a[size++] = value;
+  }
+
+  public long get( int idx )
+  {
+    if ( idx >= size )
+    {
+      throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
+    }
+    return a[idx];
+  }
+
+  public int size()
+  {
+    return size;
+  }
+
+}

brouter-util/src/main/java/btools/util/TinyDenseLongMap.java

@@ -0,0 +1,206 @@
+package btools.util;
+
+/**
+ * TinyDenseLongMap implements the DenseLongMap interface
+ * but actually is made for a medium count of non-dense keys
+ * 
+ * It's used as a replacement for DenseLongMap where we
+ * have limited memory and far less keys than maykey
+ *
+ * @author ab
+ */
+public class TinyDenseLongMap extends DenseLongMap
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+
+  public TinyDenseLongMap()
+  {
+    super(1);
+
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    // same for the values
+    vla = new byte[MAXLISTS][];
+    vla[0] = new byte[1];
+  }
+
+
+  private byte[][] vla; // value list array
+
+  private void fillReturnValue(byte[] rv, int idx, int p )
+  {
+    rv[0] = vla[idx][p];
+    if ( rv.length == 2 )
+    {
+      vla[idx][p] = rv[1];
+    }
+  }
+
+  @Override
+  public void put( long id, int value )
+  {
+    byte[] rv = new byte[2];
+    rv[1] = (byte)value;
+    if ( contains( id, rv ) )
+    {
+      return;
+    }
+
+    vla[0][0] = (byte)value;
+    _add( id );
+  }
+
+
+  /**
+   * Get the byte for the given id
+   * @param id the key to query
+   * @return the object
+   * @exception IllegalArgumentException if id is unknown
+   */
+  @Override
+  public int getInt( long id )
+  {
+    byte[] rv = new byte[1];
+    if ( contains( id, rv ) )
+    {
+      return rv[0];
+    }
+    return -1;
+  }
+
+
+  private boolean _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+      vla[idx] = new byte[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+      vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+      vla[idx] = null;
+    }
+
+    return false;
+  }
+
+
+  private boolean contains( long id, byte[] rv )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id, rv ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id, byte[] rv )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      if ( rv != null )
+      {
+        fillReturnValue( rv, idx, n );
+      }
+      return true;
+    }
+    return false;
+  }
+
+}

brouter-util/src/test/java/btools/util/CompactLongMap.java

@@ -0,0 +1,328 @@
+package btools.util;
+
+import java.util.ArrayList;
+
+/**
+ * Memory efficient Map to map a long-key to an object-value
+ *
+ * Implementation is such that basically the 12 bytes
+ * per entry is allocated that's needed to store
+ * a long- and an object-value.
+ * This class does not implement the Map interface
+ * because it's not complete (remove() is not implemented,
+ * CompactLongMap can only grow.)
+ *
+ * @author ab
+ */
+public class CompactLongMap<V>
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  private V value_in;
+  protected V value_out;
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+  private static boolean earlyDuplicateCheck;
+
+  public CompactLongMap()
+  {
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    // same for the values
+    vla = new Object[MAXLISTS][];
+    vla[0] = new Object[1];
+
+    earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
+  }
+
+
+  /*
+   *
+   * The Map extension:
+   * next 5 protected methods are needed to implement value-support
+   * overwrite them all to support value structures other than the
+   * long-values implemented here as a sample.
+   *
+   * Furthermore, put() and get() method need to be implemented
+   * to access the values.
+   *
+   * Note that this map does not behave exactly like java.util.Map
+   * - put(..) with already existing key throws exception
+   * - get(..) with non-existing key thros exception
+   *
+   * If you have keys that cannot easily be mapped on long's, use
+   * a hash-function to do the mapping. But note that, in comparison
+   * to java.util.HashMap, in that case the keys itself are not saved,
+   * only the hash-values, so you need to be sure that random duplicate
+   * hashs are either excluded by the structure of your data or that
+   * you can handle the possible IllegalArgumentException
+   *
+   */
+
+  private Object[][] vla; // value list array
+
+
+  public boolean put( long id, V value )
+  {
+    try
+    {    
+      value_in = value;
+      if ( contains( id, true ) )
+      {
+        return true;
+      }
+      vla[0][0] = value;
+      _add( id );
+      return false;
+    }
+    finally
+    {
+      value_in = null;
+      value_out = null;
+    }
+  }
+
+  /**
+   * Same as put( id, value ) but duplicate check
+   * is skipped for performance. Be aware that you
+   * can get a duplicate exception later on if the
+   * map is restructured!
+   * with System parameter earlyDuplicateCheck=true you
+   * can enforce the early duplicate check for debugging
+   *
+   * @param id the key to insert
+   * @param value the value to insert object
+   * @exception IllegalArgumentException for duplicates if enabled
+   */
+  public void fastPut( long id, V value )
+  {
+    if ( earlyDuplicateCheck && contains( id ) )
+    {
+      throw new IllegalArgumentException( "duplicate key found in early check: " + id );
+    }
+    vla[0][0] = value;
+    _add( id );
+  }
+
+  /**
+   * Get the value for the given id
+   * @param id the key to query
+   * @return the object, or null if id not known
+   */
+  public V get( long id )
+  {
+    try
+    {
+      if ( contains( id, false ) )
+      {
+        return value_out;
+      }
+      return null;
+    }
+    finally
+    {
+      value_out = null;
+    }
+  }
+
+
+  /**
+   * @return the number of entries in this map
+   */
+  public int size()
+  {
+    return size;
+  }
+
+
+  private boolean _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+      vla[idx] = new Object[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+      vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+      vla[idx] = null;
+    }
+
+    return false;
+  }
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  public boolean contains( long id )
+  {
+    try
+    {
+      return contains( id, false );
+    }
+    finally
+    {
+      value_out = null;
+    }
+  }
+
+  protected boolean contains( long id, boolean doPut )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id, doPut ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id, boolean doPut )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      value_out = (V)vla[idx][n];
+      if ( doPut ) vla[idx][n] = value_in;
+      return true;
+    }
+    return false;
+  }
+
+  protected void moveToFrozenArrays( long[] faid, ArrayList<V> flv )
+  {
+    for( int i=1; i<MAXLISTS; i++ )
+    {
+      pa[i] = 0;
+    }
+
+    for( int ti = 0; ti < size; ti++ ) // target-index
+    {
+      int bp = size; // treat size as bitpattern
+      int minIdx = -1;
+      long minId = 0;
+      int idx = 1;
+      while ( bp != 0 )
+      {
+        if ( (bp&1) == 1 )
+        {
+          int p = pa[idx];
+          if ( p < al[idx].length )
+          {
+            long currentId = al[idx][p];
+            if ( minIdx < 0 || currentId < minId )
+            {
+              minIdx = idx;
+              minId = currentId;
+            }
+          }
+        }
+        idx++;
+        bp >>= 1;
+      }
+      faid[ti] = minId;
+      flv.add( (V)vla[minIdx][pa[minIdx]] );
+      pa[minIdx]++;
+
+      if ( ti > 0 && faid[ti-1] == minId )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
+      }
+    }
+
+    // free the non-frozen arrays
+    al = null;
+    vla = null;
+  }
+
+}

brouter-util/src/test/java/btools/util/CompactLongSet.java

@@ -0,0 +1,220 @@
+package btools.util;
+
+/**
+ * Memory efficient Set for long-keys
+ *
+ * @author ab
+ */
+public class CompactLongSet
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+  private static boolean earlyDuplicateCheck;
+
+  public CompactLongSet()
+  {
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    earlyDuplicateCheck = Boolean.getBoolean( "earlyDuplicateCheck" );
+  }
+
+
+  /**
+   * @return the number of entries in this set
+   */
+  public int size()
+  {
+    return size;
+  }
+
+  /**
+   * add a long value to this set if not yet in.
+   * @param id the value to add to this set.
+   * @return true if "id" already contained in this set.
+   */
+  public boolean add( long id )
+  {
+    if ( contains( id ) )
+    {
+      return true;
+    }
+    _add( id );
+    return false;
+  }
+
+  public void fastAdd( long id )
+  {
+    _add( id );
+  }
+
+  private void _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+    }
+  }
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  public boolean contains( long id )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      return true;
+    }
+    return false;
+  }
+
+  protected void moveToFrozenArray( long[] faid )
+  {
+    for( int i=1; i<MAXLISTS; i++ )
+    {
+      pa[i] = 0;
+    }
+
+    for( int ti = 0; ti < size; ti++ ) // target-index
+    {
+      int bp = size; // treat size as bitpattern
+      int minIdx = -1;
+      long minId = 0;
+      int idx = 1;
+      while ( bp != 0 )
+      {
+        if ( (bp&1) == 1 )
+        {
+          int p = pa[idx];
+          if ( p < al[idx].length )
+          {
+            long currentId = al[idx][p];
+            if ( minIdx < 0 || currentId < minId )
+            {
+              minIdx = idx;
+              minId = currentId;
+            }
+          }
+        }
+        idx++;
+        bp >>= 1;
+      }
+      faid[ti] = minId;
+      pa[minIdx]++;
+
+      if ( ti > 0 && faid[ti-1] == minId )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + minId );
+      }
+    }
+
+    // free the non-frozen array
+    al = null;
+  }
+
+}

brouter-util/src/test/java/btools/util/DenseLongMap.java

@@ -0,0 +1,140 @@
+package btools.util;
+
+import java.util.ArrayList;
+
+/**
+ * Special Memory efficient Map to map a long-key to
+ * a "small" value (some bits only) where it is expected
+ * that the keys are dense, so that we can use more or less
+ * a simple array as the best-fit data model (except for
+ * the 32-bit limit of arrays!)
+ *
+ * Additionally, to enable small-memory unit testing 
+ * of code using this map this one has a fallback for a small map
+ * where we have only few keys, but not dense. In this
+ * case, we use the mechanics of the CompactLongMap
+ *
+ * Target application are osm-node ids which are in the
+ * range 0...3 billion and basically dense (=only few
+ * nodes deleted)
+ *
+ * @author ab
+ */
+public class DenseLongMap
+{
+  private ArrayList<int[]> blocklist = new ArrayList<int[]>(1024);
+
+  private static final int BLOCKSIZE = 0x10000; // 64k * 32 bits
+  private int valuebits;
+  private int maxvalue;
+  private long maxkey;
+  private long maxmemory;
+
+  /**
+   * Creates a DenseLongMap for the given value range
+   * Note that one value is reserved for the "unset" state,
+   * so with 6 value bits you can store values in the
+   * range 0..62 only
+   *
+   * @param valuebits number of bits to use per value
+   */
+  public DenseLongMap( int valuebits )
+  {
+    if ( valuebits < 1 || valuebits > 32 )
+    {
+      throw new IllegalArgumentException( "invalid valuebits (1..32): " + valuebits );
+    }
+    this.valuebits = valuebits;
+    maxmemory = (Runtime.getRuntime().maxMemory() / 8) * 7; // assume most of it for our map
+    maxvalue = (1 << valuebits) - 2;
+    maxkey = ( maxmemory / valuebits ) * 8;
+  }
+
+
+
+  public void put( long key, int value )
+  {
+    if ( key < 0L || key > maxkey )
+    {
+      throw new IllegalArgumentException( "key out of range (0.." + maxkey + "): " + key
+            + " give more memory (currently " + (maxmemory / 0x100000)
+            + "MB) to extend key range" );
+    }
+    if ( value < 0 || value > maxvalue )
+    {
+      throw new IllegalArgumentException( "value out of range (0.." + maxvalue + "): " + value );
+    }
+
+    int blockn = (int)(key >> 21);
+    int offset = (int)(key & 0x1fffff);
+
+    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+
+    if ( block == null )
+    {
+      block = new int[BLOCKSIZE * valuebits];
+
+      while (blocklist.size() < blockn+1 )
+      {
+        blocklist.add(null);
+      }      
+      blocklist.set( blockn, block );
+    }
+    
+    int bitmask = 1 << (offset & 0x1f);
+    int invmask = bitmask ^ 0xffffffff;
+    int probebit = 1;
+    int blockidx = (offset >> 5)*valuebits;
+    int blockend = blockidx + valuebits;
+    int v = value + 1; // 0 is reserved (=unset)
+
+    while( blockidx < blockend )
+    {
+      if ( ( v & probebit ) != 0 )
+      {
+        block[blockidx] |= bitmask;
+      }
+      else
+      {
+        block[blockidx] &= invmask;
+      }
+      probebit <<= 1;
+      blockidx++;
+    }
+  }
+
+
+  public int getInt( long key )
+  {
+    if ( key < 0 )
+    {
+      return -1;
+    }
+    int blockn = (int)(key >> 21);
+    int offset = (int)(key & 0x1fffff);
+
+    int[] block = blockn < blocklist.size() ? blocklist.get( blockn ) : null;
+
+    if ( block == null )
+    {
+      return -1;
+    }
+    int bitmask = 1 << (offset & 0x1f);
+    int probebit = 1;
+    int blockidx = (offset >> 5)*valuebits;
+    int blockend = blockidx + valuebits;
+    int v = 0; // 0 is reserved (=unset)
+
+    while( blockidx < blockend )
+    {
+      if ( ( block[blockidx] & bitmask ) != 0 )
+      {
+        v  |= probebit;
+      }
+      probebit <<= 1;
+      blockidx++;
+    }
+    return v-1;
+  }
+
+}

brouter-util/src/test/java/btools/util/FrozenLongMap.java

@@ -0,0 +1,122 @@
+package btools.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Frozen instance of Memory efficient Map
+ *
+ * This one is readily sorted into a singe array for faster access
+ *
+ * @author ab
+ */
+public class FrozenLongMap<V> extends CompactLongMap<V>
+{
+  private long[] faid;
+  private ArrayList<V> flv;
+  private int size = 0;
+  private int p2size; // next power of 2 of size
+
+  public FrozenLongMap( CompactLongMap<V> map )
+  {
+    size = map.size();
+
+    faid = new long[size];
+    flv = new ArrayList<V>(size);
+
+    map.moveToFrozenArrays( faid, flv );
+
+    p2size = 0x40000000;
+    while( p2size > size ) p2size >>= 1;
+  }
+
+  @Override
+  public boolean put( long id, V value )
+  {
+    throw new RuntimeException( "cannot put on FrozenLongIntMap" );
+  }
+
+  @Override
+  public void fastPut( long id, V value )
+  {
+    throw new RuntimeException( "cannot put on FrozenLongIntMap" );
+  }
+
+  /**
+   * @return the number of entries in this set
+   */
+  @Override
+  public int size()
+  {
+    return size;
+  }
+
+
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  @Override
+  protected boolean contains( long id, boolean doPut )
+  {
+    if ( size == 0 )
+    {
+      return false;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      value_out = flv.get(n);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * @return the value for "id",
+   * Throw an exception if not contained in the map.
+   */
+  @Override
+  public V get( long id )
+  {
+    if ( size == 0 )
+    {
+      return null;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      return flv.get(n);
+    }
+    return null;
+  }
+
+  public List<V> getValueList()
+  {
+    return flv;
+  }
+}

brouter-util/src/test/java/btools/util/FrozenLongSet.java

@@ -0,0 +1,81 @@
+package btools.util;
+
+/**
+ * Frozen instance of Memory efficient Set
+ *
+ * This one is readily sorted into a singe array for faster access
+ *
+ * @author ab
+ */
+public class FrozenLongSet extends CompactLongSet
+{
+  private long[] faid;
+  private int size = 0;
+  private int p2size; // next power of 2 of size
+
+  public FrozenLongSet( CompactLongSet set )
+  {
+    size = set.size();
+
+    faid = new long[size];
+
+    set.moveToFrozenArray( faid );
+
+    p2size = 0x40000000;
+    while( p2size > size ) p2size >>= 1;
+  }
+
+  @Override
+  public boolean add( long id )
+  {
+    throw new RuntimeException( "cannot add on FrozenLongSet" );
+  }
+
+  @Override
+  public void fastAdd( long id )
+  {
+    throw new RuntimeException( "cannot add on FrozenLongSet" );
+  }
+
+  /**
+   * @return the number of entries in this set
+   */
+  @Override
+  public int size()
+  {
+    return size;
+  }
+
+
+
+  /**
+   * @return true if "id" is contained in this set.
+   */
+  @Override
+  public boolean contains( long id )
+  {
+    if ( size == 0 )
+    {
+      return false;
+    }
+    long[] a = faid;
+    int offset = p2size;
+    int n = 0;
+
+    while ( offset> 0 )
+    {
+      int nn = n + offset;
+      if ( nn < size && a[nn] <= id )
+      {
+        n = nn;
+      }
+      offset >>= 1;
+    }
+    if ( a[n] == id )
+    {
+      return true;
+    }
+    return false;
+  }
+
+}

brouter-util/src/test/java/btools/util/LazyArrayOfLists.java

@@ -0,0 +1,53 @@
+package btools.util;
+
+import java.util.List;
+import java.util.ArrayList;
+
+/**
+ * Behaves like an Array of list
+ * with lazy list-allocation at getList
+ *
+ * @author ab
+ */
+public class LazyArrayOfLists<E>
+{
+  private ArrayList<ArrayList<E>> lists;
+
+  public LazyArrayOfLists( int size )
+  {
+    lists = new ArrayList<ArrayList<E>>( size );
+    for ( int i = 0; i< size; i++ )
+    {
+      lists.add( null );
+    }
+  }
+
+  public List<E> getList( int idx )
+  {
+    ArrayList<E> list = lists.get( idx );
+    if ( list == null )
+    {
+      list = new ArrayList<E>();
+      lists.set( idx, list );
+    }
+    return list;
+  }
+
+  public int getSize( int idx )
+  {
+    List<E> list = lists.get( idx );
+    return list == null ? 0 : list.size();
+  }
+    
+  public void trimAll()
+  {
+    for ( int idx = 0; idx< lists.size(); idx++ )
+    {
+      ArrayList<E> list = lists.get( idx );
+      if ( list != null )
+      {
+        list.trimToSize();
+      }
+    }
+  }
+}

brouter-util/src/test/java/btools/util/LongList.java

@@ -0,0 +1,43 @@
+package btools.util;
+
+/**
+ * dynamic list of primitive longs
+ *
+ * @author ab
+ */
+public class LongList
+{
+  private long[] a;
+  private int size;
+
+  public LongList( int capacity )
+  {
+    a = capacity < 4 ? new long[4] : new long[capacity];
+  }
+
+  public void add( long value )
+  {
+    if ( size == a.length )
+    {
+      long[] aa = new long[2*size];
+      System.arraycopy( a, 0, aa, 0, size );
+      a = aa;
+    }
+    a[size++] = value;
+  }
+
+  public long get( int idx )
+  {
+    if ( idx >= size )
+    {
+      throw new IndexOutOfBoundsException( "list size=" + size + " idx=" + idx );
+    }
+    return a[idx];
+  }
+
+  public int size()
+  {
+    return size;
+  }
+
+}

brouter-util/src/test/java/btools/util/TinyDenseLongMap.java

@@ -0,0 +1,206 @@
+package btools.util;
+
+/**
+ * TinyDenseLongMap implements the DenseLongMap interface
+ * but actually is made for a medium count of non-dense keys
+ * 
+ * It's used as a replacement for DenseLongMap where we
+ * have limited memory and far less keys than maykey
+ *
+ * @author ab
+ */
+public class TinyDenseLongMap extends DenseLongMap
+{
+  private long[][] al;
+  private int[] pa;
+  private int size = 0;
+  private int _maxKeepExponent = 14; // the maximum exponent to keep the invalid arrays
+
+  protected static final int MAXLISTS = 31; // enough for size Integer.MAX_VALUE
+
+  public TinyDenseLongMap()
+  {
+    super(1);
+
+    // pointer array
+    pa = new int[MAXLISTS];
+
+    // allocate key lists
+    al = new long[MAXLISTS][];
+    al[0] = new long[1]; // make the first array (the transient buffer)
+
+    // same for the values
+    vla = new byte[MAXLISTS][];
+    vla[0] = new byte[1];
+  }
+
+
+  private byte[][] vla; // value list array
+
+  private void fillReturnValue(byte[] rv, int idx, int p )
+  {
+    rv[0] = vla[idx][p];
+    if ( rv.length == 2 )
+    {
+      vla[idx][p] = rv[1];
+    }
+  }
+
+  @Override
+  public void put( long id, int value )
+  {
+    byte[] rv = new byte[2];
+    rv[1] = (byte)value;
+    if ( contains( id, rv ) )
+    {
+      return;
+    }
+
+    vla[0][0] = (byte)value;
+    _add( id );
+  }
+
+
+  /**
+   * Get the byte for the given id
+   * @param id the key to query
+   * @return the object
+   * @exception IllegalArgumentException if id is unknown
+   */
+  @Override
+  public int getInt( long id )
+  {
+    byte[] rv = new byte[1];
+    if ( contains( id, rv ) )
+    {
+      return rv[0];
+    }
+    return -1;
+  }
+
+
+  private boolean _add( long id )
+  {
+    if ( size == Integer.MAX_VALUE )
+    {
+      throw new IllegalArgumentException( "cannot grow beyond size Integer.MAX_VALUE" );
+    }
+
+    // put the new entry in the first array
+    al[0][0] = id;
+
+    // determine the first empty array
+    int bp = size++; // treat size as bitpattern
+    int idx = 1;
+    int n = 1;
+
+    pa[0] = 1;
+    pa[1] = 1;
+
+    while ( (bp&1) == 1 )
+    {
+      bp >>= 1;
+      pa[idx++] = n;
+      n <<= 1;
+    }
+
+    // create it if not existant
+    if ( al[idx] == null )
+    {
+      al[idx] = new long[n];
+      vla[idx] = new byte[n];
+    }
+
+    // now merge the contents of arrays 0...idx-1 into idx
+    while ( n > 0 )
+    {
+      long maxId = 0;
+      int maxIdx = -1;
+
+      for ( int i=0; i<idx; i++ )
+      {
+        int p = pa[i];
+        if ( p > 0 )
+        {
+          long currentId = al[i][p-1];
+          if ( maxIdx < 0 || currentId > maxId )
+          {
+            maxIdx = i;
+            maxId = currentId;
+          }
+        }
+      }
+
+      // current maximum found, copy to target array
+      if ( n < al[idx].length && maxId == al[idx][n] )
+      {
+        throw new IllegalArgumentException( "duplicate key found in late check: " + maxId );
+      }
+      --n;
+      al[idx][n] = maxId;
+      vla[idx][n] = vla[maxIdx][pa[maxIdx]-1];
+
+      --pa[maxIdx];
+    }
+
+    // de-allocate empty arrays of a certain size (fix at 64kByte)
+    while ( idx-- > _maxKeepExponent )
+    {
+      al[idx] = null;
+      vla[idx] = null;
+    }
+
+    return false;
+  }
+
+
+  private boolean contains( long id, byte[] rv )
+  {
+    // determine the first empty array
+    int bp = size; // treat size as bitpattern
+    int idx = 1;
+
+    while ( bp != 0 )
+    {
+      if ( (bp&1) == 1 )
+      {
+        // array at idx is valid, check
+        if ( contains( idx, id, rv ) )
+        {
+          return true;
+        }
+      }
+      idx++;
+      bp >>= 1;
+    }
+    return false;
+  }
+
+
+  // does sorted array "a" contain "id" ?
+  private boolean contains( int idx, long id, byte[] rv )
+  {
+    long[] a = al[idx];
+    int offset = a.length;
+    int n = 0;
+
+    while ( (offset >>= 1) > 0 )
+    {
+      int nn = n + offset;
+      if ( a[nn] <= id )
+      {
+        n = nn;
+      }
+    }
+    if ( a[n] == id )
+    {
+      if ( rv != null )
+      {
+        fillReturnValue( rv, idx, n );
+      }
+      return true;
+    }
+    return false;
+  }
+
+}