Project Description
parse logs with LINQ
detailed description will be later

Current Limitations

1. for now supports only iis w3c logs
2. Slice methods work very fast (binary search), but don't support files with changes in columns list
3. requires admin rights
4. for grouping with very big size of groups x64 buid required (linq using simple lists for this)

Demo

1. all *.htm requests
from a in Site(1)
where a.HaveExtension(".htm")
select new {Date=a.DateTime, Uri=a.Uri.AbsolutePath}

2. all ips between 192.168.0.0 and 192.168.255.255 from specific date
from a in Site(1).Slice(new DateTime(2013, 1, 1))
where a.ClientIP>IPAddress.Parse("192.168.0.0") && a.ClientIP<IPAddress.Parse("192.168.255.255")
group a by a.ClientIP into g
select new{g.Key, Count=g.Count()}

3. all users from 2013-01-01 1:00 to 2013-01-01 2:00
(from a in Site(1).Slice(new DateTime(2013, 1, 1, 1, 0, 0), new DateTime(2013, 1, 1, 2, 0, 0))
select a.UserName).Distinct()

4. hits by extensions for last 4 hours
from a in Site(1).Slice(TimeSpan.FromHours(4))
let ex=a.GetExtension().ToUpper()
group a by ex into g
let count=g.Count()
orderby count descending
select new{g.Key, count}

5. total sent bytes and hits
new{sent=Site(1).Sum(a=>a.SentBytes), hits=Site(1).Count()}

6. all error requests by status
from a in Site(1)
where a.Status>=400
group a by new {a.Uri.PathAndQuery, Status=a.Status+"."+a.SubStatus} into g
let count=g.Count()
orderby count descending
select new{g.Key.PathAndQuery, g.Key.Status, count}

7. helper against simple ddos attacks or against heavy load
var siteNumber=1;
var time=TimeSpan.FromMinutes(15);

(from a in Site(siteNumber).Slice(time)
group a by a.ClientIP into g
let count=g.Count()
orderby count descending
select new{IP=g.Key, Count=count, TimeTaken=TimeSpan.FromMilliseconds(g.Sum(b=>b.TimeTaken.TotalMilliseconds)),
	ReceivedMegaBytes=Math.Round((double)g.Sum(b=>b.ReceivedBytes)/1024/1024, 3), SentMegaBytes=Math.Round((double)g.Sum(b=>b.SentBytes)/1024/1024, 3),
	Requests=
		from c in g
	    group c by c.Uri.AbsolutePath into g1
		let count1=g1.Count()
	    orderby count1 descending
	    select new{URI=g1.Key, Count=count1, TimeTaken=TimeSpan.FromMilliseconds(g1.Sum(b=>b.TimeTaken.TotalMilliseconds)),
        	ReceivedMegaBytes=Math.Round((double)g1.Sum(b=>b.ReceivedBytes)/1024/1024, 3), SentMegaBytes=Math.Round((double)g1.Sum(b=>b.SentBytes)/1024/1024, 3)},
	RequestsWithQuery=
		from c in g
		group c by c.Uri.PathAndQuery into g1
		let count1=g1.Count()
		orderby count1 descending
		select new{URI=g1.Key, Count=count1, TimeTaken=TimeSpan.FromMilliseconds(g1.Sum(b=>b.TimeTaken.TotalMilliseconds)),
        	ReceivedMegaBytes=Math.Round((double)g1.Sum(b=>b.ReceivedBytes)/1024/1024, 3), SentMegaBytes=Math.Round((double)g1.Sum(b=>b.SentBytes)/1024/1024, 3)}
}).Dump("by IP", true);


(from a in Site(siteNumber).Slice(time)
group a by a.Uri.AbsolutePath into g
let count=g.Count()
orderby count descending
select new{URI=g.Key, Count=count, TimeTaken=TimeSpan.FromMilliseconds(g.Sum(b=>b.TimeTaken.TotalMilliseconds)),
	ReceivedMegaBytes=Math.Round((double)g.Sum(b=>b.ReceivedBytes)/1024/1024, 3), SentMegaBytes=Math.Round((double)g.Sum(b=>b.SentBytes)/1024/1024, 3),
	Requests=
		from c in g
	    group c by c.ClientIP into g1
		let count1=g1.Count()
	    orderby count1 descending
	    select new{URI=g1.Key, Count=count1, TimeTaken=TimeSpan.FromMilliseconds(g1.Sum(b=>b.TimeTaken.TotalMilliseconds)),
        	ReceivedMegaBytes=Math.Round((double)g1.Sum(b=>b.ReceivedBytes)/1024/1024, 3), SentMegaBytes=Math.Round((double)g1.Sum(b=>b.SentBytes)/1024/1024, 3)}
}).Dump("by URI", true);
First you can view requests grouped by ip or URI
1.jpg
By clicking to link you can view all requests (loaded on demand) to this IP/URI (simple ddos attacks don't requests page resoures (images, scripts, css))
2.jpg

Last edited Jul 7, 2013 at 7:48 PM by pr0ff, version 8