ben/brouter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into cleanup-fileaccess

Browse source
This commit is contained in:
afischerdev 2021-07-01 11:20:30 +02:00 • committed by GitHub
commit 8c5a961011
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 1637 additions and 1084 deletions
Download patch file Download diff file Expand all files Collapse all files

52
brouter-server/src/main/java/btools/server/IpAccessMonitor.java Normal file
View file

@ -0,0 +1,52 @@
package btools.server;
import java.util.HashMap;
import java.util.Map;
public class IpAccessMonitor
{
private static Object sync = new Object();
private static HashMap<String,Long> ipAccess = new HashMap<String,Long>();
private static long MAX_IDLE = 900000; // 15 minutes
private static long CLEANUP_INTERVAL = 10000; // 10 seconds
private static long lastCleanup;
public static boolean touchIpAccess( String ip )
{
long t = System.currentTimeMillis();
synchronized( sync )
{
Long lastTime = ipAccess.get( ip );
ipAccess.put( ip, Long.valueOf( t ) );
return lastTime == null || t - lastTime.longValue() > MAX_IDLE;
}
}
public static int getSessionCount()
{
long t = System.currentTimeMillis();
synchronized( sync )
{
if ( t - lastCleanup > CLEANUP_INTERVAL )
{
cleanup( t );
lastCleanup = t;
}
return ipAccess.size();
}
}
private static void cleanup( long t )
{
HashMap<String,Long> newMap = new HashMap<String,Long>(ipAccess.size());
for( Map.Entry<String,Long> e : ipAccess.entrySet() )
{
if ( t - e.getValue().longValue() <= MAX_IDLE )
{
newMap.put( e.getKey(), e.getValue() );
}
}
ipAccess = newMap;
}
}

69
brouter-server/src/main/java/btools/server/RouteServer.java
View file

@ -61,7 +61,7 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
private static DateFormat tsFormat = new SimpleDateFormat( "dd.MM.yy HH:mm", new Locale( "en", "US" ) );
private static String formattedTimestamp()
private static String formattedTimeStamp( long t )
{
synchronized( tsFormat )
{
@ -73,15 +73,21 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
{
BufferedReader br = null;
BufferedWriter bw = null;
// first line
String getline = null;
String sessionInfo = null;
String sIp = null;
try
{
br = new BufferedReader( new InputStreamReader( clientSocket.getInputStream() , "UTF-8") );
bw = new BufferedWriter( new OutputStreamWriter( clientSocket.getOutputStream(), "UTF-8" ) );
// first line
String getline = null;
String agent = null;
String encodings = null;
String xff = null; // X-Forwarded-For
String referer = null;
// more headers until first empty line
for(;;)
@ -102,16 +108,40 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
{
getline = line;
}
if ( line.startsWith( "User-Agent: " ) )
line = line.toLowerCase();
if ( line.startsWith( "user-agent: " ) )
{
agent = line.substring( "User-Agent: ".length() );
agent = line.substring( "user-agent: ".length() );
}
if ( line.startsWith( "Accept-Encoding: " ) )
if ( line.startsWith( "accept-encoding: " ) )
{
encodings = line.substring( "Accept-Encoding: ".length() );
encodings = line.substring( "accept-encoding: ".length() );
}
if ( line.startsWith( "x-forwarded-for: " ) )
{
xff = line.substring( "x-forwarded-for: ".length() );
}
if ( line.startsWith( "Referer: " ) )
{
referer = line.substring( "Referer: ".length() );
}
if ( line.startsWith( "Referrer: " ) )
{
referer = line.substring( "Referrer: ".length() );
}
}
InetAddress ip = clientSocket.getInetAddress();
sIp = xff == null ? (ip==null ? "null" : ip.toString() ) : xff;
boolean newSession = IpAccessMonitor.touchIpAccess( sIp );
sessionInfo = " new";
if ( !newSession )
{
int sessionCount = IpAccessMonitor.getSessionCount();
sessionInfo = " " + Math.min( sessionCount, 999 );
sessionInfo = sessionInfo.substring( sessionInfo.length() - 4 );
}
String excludedAgents = System.getProperty( "excludedAgents" );
if ( agent != null && excludedAgents != null )
{
@ -128,6 +158,17 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
}
}
if ( referer != null && referer.indexOf( "brouter.de/brouter-web" ) >= 0 )
{
if ( getline.indexOf( "%7C" ) >= 0 && getline.indexOf( "%2C" ) >= 0 )
{
writeHttpHeader( bw, HTTP_STATUS_FORBIDDEN );
bw.write( "Spam? please stop" );
bw.flush();
return;
}
}
if ( getline.startsWith("GET /favicon.ico") )
{
writeHttpHeader( bw, HTTP_STATUS_NOT_FOUND );
@ -143,9 +184,6 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
return;
}
InetAddress ip = clientSocket.getInetAddress();
System.out.println( formattedTimestamp() + " ip=" + (ip==null ? "null" : ip.toString() ) + " -> " + getline );
String url = getline.split(" ")[1];
HashMap<String,String> params = getUrlParams(url);
@ -280,8 +318,12 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
{
threadPoolSync.notifyAll();
}
long t = System.currentTimeMillis();
long ms = t - starttime;
System.out.println( formattedTimeStamp(t) + sessionInfo + " ip=" + sIp + " ms=" + ms + " -> " + getline );
}
}
public static void main(String[] args) throws Exception
{
@ -355,13 +397,18 @@ public class RouteServer extends Thread implements Comparable<RouteServer>
{
threadPoolSync.wait( maxWaitTime );
}
long t = System.currentTimeMillis();
System.out.println( formattedTimeStamp(t) + " contention! ms waited " + (t - server.starttime) );
}
cleanupThreadQueue( threadQueue );
if ( threadQueue.size() >= maxthreads )
{
if ( debug ) System.out.println( "stopping oldest thread..." );
// no way... stop the oldest thread
threadQueue.poll().stopRouter();
RouteServer oldest = threadQueue.poll();
oldest.stopRouter();
long t = System.currentTimeMillis();
System.out.println( formattedTimeStamp(t) + " contention! ms killed " + (t - oldest.starttime) );
}
}

3
brouter-server/src/main/java/btools/server/SuspectManager.java
View file

@ -450,6 +450,8 @@ public class SuspectManager extends Thread
String url5 = "https://tyrasd.github.io/latest-changes/#16/" + dlat + "/" + dlon;
String url6 = "https://apps.sentinel-hub.com/sentinel-playground/?source=S2L2A&lat=" + dlat + "&lng=" + dlon + "&zoom=15";
if ( message != null )
{
bw.write( "<strong>" + message + "</strong><br><br>\n" );
@ -459,6 +461,7 @@ public class SuspectManager extends Thread
bw.write( "<a href=\"" + url3 + "\">Open in JOSM (via remote control)</a><br><br>\n" );
bw.write( "Overpass: <a href=\"" + url4a + "\">minus one week</a> &nbsp;&nbsp; <a href=\"" + url4b + "\">node context</a><br><br>\n" );
bw.write( "<a href=\"" + url5 + "\">Open in Latest-Changes / last week</a><br><br>\n" );
bw.write( "<a href=\"" + url6 + "\">Current Sentinel-2 imagary</a><br><br>\n" );
bw.write( "<br>\n" );
if ( isFixed( id, suspects.timestamp ) )
{

2
brouter-server/src/test/java/btools/server/IntegrityCheckTest.java
View file

@ -20,7 +20,7 @@ public class IntegrityCheckTest
File resultfile = new File( resulturl.getFile() );
workingDir = resultfile.getParentFile();
File segmentDir = new File( workingDir, "/../../../brouter-map-creator/target/test-classes/tmp/segments" );
File segmentDir = new File( workingDir, "/../../../../brouter-map-creator/build/resources/test/tmp/segments" );
File[] files = segmentDir.listFiles();
for ( File f : files )

7
brouter-server/src/test/java/btools/server/RouterTest.java
View file

@ -61,13 +61,16 @@ public class RouterTest
wplist.add( n );
RoutingContext rctx = new RoutingContext();
rctx.localFunction = wd + "/../../../misc/profiles2/trekking.brf";
rctx.localFunction = wd + "/../../../../misc/profiles2/trekking.brf";
// c.setAlternativeIdx( 1 );
RoutingEngine re = new RoutingEngine(
wd + "/" + trackname,
wd + "/" + trackname,
new File ( wd, "/../../../brouter-map-creator/target/test-classes/tmp/segments"), wplist, rctx );
new File ( wd, "/../../../../brouter-map-creator/build/resources/test/tmp/segments"),
wplist,
rctx );
re.doRun( 0 );
return re.getErrorMessage();