ben/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

rd5 diff progress

Browse source
This commit is contained in:
Arndt Brenschede 2020-01-19 15:22:05 +01:00
parent db4757abbb
commit 43de4bb6ca
4 changed files with 227 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

19
brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffManager.java
View file

@ -48,6 +48,8 @@ final public class Rd5DiffManager
// calculate MD5 of old file
String md5 = getMD5( fo );
String md5New = getMD5( fn );
System.out.println( "name=" + name + " md5=" + md5 );
@ -55,9 +57,12 @@ final public class Rd5DiffManager
specificNewDiffs.mkdirs();
String diffFileName = md5 + ".rd5diff";
File diffFile = new File( specificNewDiffs, diffFileName );
String dummyDiffFileName = md5New + ".rd5diff";
File dummyDiffFile = new File( specificNewDiffs, dummyDiffFileName );
dummyDiffFile.createNewFile();
// calc the new diff
Rd5DiffTool.diff2files( fo, fn, diffFile );
@ -72,15 +77,17 @@ final public class Rd5DiffManager
{
continue;
}
if ( System.currentTimeMillis() - od.lastModified() > 31*86400000L )
if ( System.currentTimeMillis() - od.lastModified() > 9*86400000L )
{
continue; // limit diff history to 31 days
continue; // limit diff history to 9 days
}
File updatedDiff = new File( specificNewDiffs, od.getName() );
Rd5DiffTool.addDeltas( od, diffFile, updatedDiff );
updatedDiff.setLastModified( od.lastModified() );
if ( !updatedDiff.exists() )
{
Rd5DiffTool.addDeltas( od, diffFile, updatedDiff );
updatedDiff.setLastModified( od.lastModified() );
}
}
}
}

113
brouter-mapaccess/src/main/java/btools/mapaccess/Rd5DiffTool.java
View file

@ -19,8 +19,9 @@ import btools.codec.MicroCache;
import btools.codec.MicroCache2;
import btools.codec.StatCoderContext;
import btools.util.Crc32;
import btools.util.ProgressListener;
final public class Rd5DiffTool
final public class Rd5DiffTool implements ProgressListener
{
public static void main( String[] args ) throws Exception
{
@ -32,7 +33,7 @@ final public class Rd5DiffTool
}
else
{
recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ) /*, new File( args[3] ) */ );
recoverFromDelta( new File( args[0] ),new File( args[1] ), new File( args[2] ), new Rd5DiffTool() /*, new File( args[3] ) */ );
}
}
else
@ -41,6 +42,18 @@ final public class Rd5DiffTool
}
}
@Override
public void updateProgress( String progress )
{
System.out.println( progress );
}
@Override
public boolean isCanceled()
{
return false;
}
private static long[] readFileIndex( DataInputStream dis, DataOutputStream dos ) throws Exception
{
long[] fileIndex = new long[25];
@ -275,10 +288,20 @@ final public class Rd5DiffTool
}
public static void recoverFromDelta( File f1, File f2, File outFile /* , File cmpFile */ ) throws Exception
public static void recoverFromDelta( File f1, File f2, File outFile, ProgressListener progress /* , File cmpFile */ ) throws Exception
{
if ( f2.length() == 0L )
{
copyFile( f1, outFile, progress );
return;
}
byte[] abBuf1 = new byte[10 * 1024 * 1024];
byte[] abBuf2 = new byte[10 * 1024 * 1024];
boolean canceled = false;
long t0 = System.currentTimeMillis();
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataInputStream dis2 = new DataInputStream( new BufferedInputStream( new FileInputStream( f2 ) ) );
@ -290,6 +313,8 @@ final public class Rd5DiffTool
long[] fileIndex2 = readFileIndex( dis2, dos );
// long[] fileIndexCmp = readFileIndex( disCmp, null );
int lastPct = -1;
try
{
DataBuffers dataBuffers = new DataBuffers();
@ -307,6 +332,19 @@ final public class Rd5DiffTool
for ( int tileIdx = 0; tileIdx < 1024; tileIdx++ )
{
if ( progress.isCanceled() )
{
canceled = true;
return;
}
double bytesProcessed = getTileStart( fileIndex1, subFileIdx ) + ( posIdx1 == null ? 0 : getPosIdx( posIdx1, tileIdx-1 ) );
int pct = (int)(100. * bytesProcessed / getTileEnd( fileIndex1, 24 ) + 0.5 );
if ( pct != lastPct )
{
progress.updateProgress( "Applying delta: " + pct + "%" );
lastPct = pct;
}
byte[] ab1 = createMicroCache( posIdx1, tileIdx, dis1, false );
byte[] ab2 = createMicroCache( posIdx2, tileIdx, dis2, true );
if ( ab2 == null )
@ -359,6 +397,7 @@ final public class Rd5DiffTool
*/
dos.write( abBuf1, 0, len );
dos.writeInt( Crc32.crc( abBuf1, 0, len ) ^ 2 );
}
}
// write any remaining data to the output file
@ -371,6 +410,8 @@ final public class Rd5DiffTool
}
dos.write( abBuf1, 0, len );
}
long t1 = System.currentTimeMillis();
System.out.println( "recovering from diffs took " + (t1-t0) + "ms" );
}
finally
{
@ -403,6 +444,72 @@ final public class Rd5DiffTool
catch (Exception ee)
{
}
if ( canceled )
{
outFile.delete();
}
}
}
}
public static void copyFile( File f1, File outFile, ProgressListener progress ) throws Exception
{
boolean canceled = false;
DataInputStream dis1 = new DataInputStream( new BufferedInputStream( new FileInputStream( f1 ) ) );
DataOutputStream dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( outFile ) ) );
int lastPct = -1;
long sizeTotal = f1.length();
long sizeRead = 0L;
try
{
byte buf[] = new byte[65536];
for (;;)
{
if ( progress.isCanceled() )
{
canceled = true;
return;
}
int pct = (int)( (100. * sizeRead) / (sizeTotal+1) + 0.5 );
if ( pct != lastPct )
{
progress.updateProgress( "Copying: " + pct + "%" );
lastPct = pct;
}
int len = dis1.read( buf );
if ( len <= 0 )
{
break;
}
sizeRead += len;
dos.write( buf, 0, len );
}
}
finally
{
if ( dis1 != null )
{
try
{
dis1.close();
}
catch (Exception ee)
{
}
}
if ( dos != null )
{
try
{
dos.close();
}
catch (Exception ee)
{
}
if ( canceled )
{
outFile.delete();
}
}
}
}