Monitoring Our Build Farm — Visually

So now that I’m working full time hours, I’d like to monitor the ARM computers in our build farm more closely and in a nicer manner. These series of scripts (d3 js and Twitter Bootstrap) are based on a whole bunch of other scripts like: Koji Client Command, Icinga, and my own check_koji.pl nagios plugin. The result looks like the following:

farm.py - A python script that gets the needed info being displayed

#!/usr/bin/python

import getpass
import os
import re
import sys
import time
import subprocess
import urllib2, base64

sys.stdout.write("Username: ")
sys.stdout.flush()
user = sys.stdin.readline().strip()

spwd = getpass.unix_getpass()

while (1):
	indx = 0
	buil = {}
	node = ""
	link = ""
	
	node += ("{\"name\":\"Hosts\",\"size\":0,\"group\":0,\"info\":\"root\"}")
	link += ("{\"source\":0,\"target\":0}")
	indx += 1
	
	bsfs = base64.encodestring("%s:%s" % (user, spwd)).replace("\n", "")
	
	try:
		requ = urllib2.Request(sys.argv[1] + "/cgi-bin/status.cgi?host=all")
		requ.add_header("Authorization", "Basic %s" % bsfs)
		resu = urllib2.urlopen(requ)
		webp = resu.read().replace("\0","").replace("\t","").replace("\r","").replace("\n","")
	except:
		print("urllib...")
		time.sleep(60)
		continue
	
	webp = re.sub("<TABLE BORDER=0 CELLSPACING=0 CELLPADDING=0>", "\n<beg line='true'>", webp)
	
	for line in webp.split("\n"):
		if (re.match("^.*>Koji Stat<.*$", line, re.I)):
			host = re.sub("^.*host=([^&]+).*$", "\\1", line).strip()
			
			line = re.sub("^.+(OK|WARNING|UNKNOWN|CRITICAL).+[0-9]+[0-9]+-[0-9]+[0-9]+-[0-9]+[0-9]+[0-9]+[0-9]+[ ]+[0-9]+[0-9]+:[0-9]+[0-9]+:[0-9]+[0-9]+(.+)$", "<td>\\1</td>\\2", line).strip()
			line = line.replace("<TD", "\t<TD")
			part = line.split("\t")
			
			for x in range(0, len(part)):
				part[x] = re.sub("<[^>]*>", " ", part[x]).strip()
			
			stat = part[0]
			info = part[3]
			
			colr = 3
			
			if (stat == "CRITICAL"):
				colr = 3
			if (stat == "WARNING"):
				colr = 2
			if (stat == "OK"):
				colr = 1
				info = "OK"
			
			buil[host] = indx
			
			node += (",\n{\"name\":\"%s [%s]\",\"size\":1,\"group\":%d,\"info\":\"%s\"}" % (host, stat, colr, info))
			link += (",\n{\"source\":%d,\"target\":%d}" % (0, indx))
			indx += 1
	
	pipe = subprocess.Popen(["/usr/bin/koji", "-s", sys.argv[2], "list-tasks"], stdout=subprocess.PIPE)
	
	for line in pipe.stdout.readlines():
		line = line.replace("\t", " ").strip()
		line = re.sub(" [ ]+", " ", line)
		
		regx = re.match("^([^ ]+) .* (CANCELED|FAILED|OPEN|CLOSED) .* build[ ]+([^ ]+)[ ]+([^ ]+)$", line, re.I)
		
		if (regx):
			task = regx.group(4)
			host = regx.group(3).replace("[","").replace("]","")
			stat = regx.group(2)
			info = regx.group(1)
			colr = 4
			
			if (stat == "CANCELED"):
				colr = 3
			if (stat == "FAILED"):
				colr = 3
			if (stat == "OPEN"):
				colr = 5
			if (stat == "CLOSED"):
				colr = 1
			
			if (not host in buil.keys()):
				buil[host] = indx
				
				node += (",\n{\"name\":\"%s [%s]\",\"size\":1,\"group\":%d,\"info\":\"%s\"}" % (host, "REMOTE", 6, "N/A"))
				link += (",\n{\"source\":%d,\"target\":%d}" % (0, indx))
				indx += 1
			
			node += (",\n{\"name\":\"%s [%s]\",\"size\":1,\"group\":%d,\"info\":\"%s\"}" % (task, stat, colr, info))
			link += (",\n{\"source\":%d,\"target\":%d}" % (buil[host], indx))
			indx += 1
	
	fobj = open("force.json", "w")
	fobj.write("{\n\"nodes\":[\n%s\n],\n\"links\":[\n%s\n]\n}\n" % (node, link))
	fobj.close()
	
	print("sleeping...")
	time.sleep(45)

force.js - Some JavaScript that initializes the force directed graph

//var width = 640, height = 560, distance = 64;
var width = 1024, height = 900, distance = 128;
var sizes = [10, 10];
// blue, green, yellow, red, purple, orange, blue
var colors = ["rgba(30, 120, 180, 1)", "rgba(0, 155, 0, 1)", "rgba(250, 195, 0, 1)", "rgba(206, 22, 32, 1)", "rgba(100, 0, 155, 1)", "rgba(255, 160, 0, 1)", "rgba(30, 120, 180, 1)"];
var nodes = "", force = null, svg = null, mous = {x:0,y:0};

function over()
{
	try { var l = force.nodes(); }
	catch(err) { var l = []; }
	
	for (var i in l)
	{
		var r = sizes[l[i].size];
		var a = (l[i].x - r);
		var b = (l[i].x + r);
		var c = (l[i].y - r);
		var d = (l[i].y + r);
		
		if ((a <= mous.x) && (mous.x <= b) && (c <= mous.y) && (mous.y <= d))
		{
			document.getElementById("info").innerHTML = ("<b>" + l[i].name + "</b> :: " + l[i].info);
		}
	}
	
	setTimeout("over()", 0.15 * 1000);
}

function move(e)
{
	var obj = document.getElementsByTagName("svg");
	
	if (obj.length > 0)
	{
		var pt = obj[0].createSVGPoint();
		var scr = obj[0].getScreenCTM();
		var inv = scr.inverse();
		
		pt.x = e.clientX; pt.y = e.clientY;
		mous = pt.matrixTransform(inv);
	}
}

function tick()
{
	try { force.start(); }
	catch(err) { console.log("tick"); }
	setTimeout("tick();", 3 * 1000);
}

function loop()
{
  d3.json("force.json", function(json) {
    var stri = JSON.stringify(json);

    if (stri == nodes)
    {
      return;
    }
    //console.log(nodes+"!="+stri);

    nodes = stri;
    document.getElementById("chart").innerHTML = "";

    force = d3.layout.force()
      .charge(-512)
      .linkDistance(distance)
      .size([width, height]);

    svg = d3.select("#chart").append("svg")
      .attr("width", width)
      .attr("height", height);

    force
      .nodes(json.nodes)
      .links(json.links)
      .start();

    var l = force.nodes();
    document.getElementById("list").innerHTML = "";

    for (var i in l)
    {
      if (l[i].name.match(/^.*cdot.*(WARNING|CRITICAL|UNKNOWN).*$/i))
      {
        document.getElementById("list").innerHTML += ("<div class='alert alert-error' style='margin: 5px;'><b>" + l[i].name + "</b> :: " + l[i].info + "</div>");
      }
    }

    var link = svg.selectAll("line.link")
      .data(json.links)
      .enter().append("line")
      .attr("class", "link")
      .style("stroke-width", function(d) { return Math.sqrt(4); });

    var node = svg.selectAll("circle.node")
      .data(json.nodes)
      .enter().append("circle")
      .attr("class", "node")
      .attr("r", function(d) { return sizes[d.size]; })
      .style("fill", function(d) { return colors[d.group]; })
      .call(force.drag);

    force.on("tick", function() {
      link.attr("x1", function(d) { return d.source.x; })
        .attr("y1", function(d) { return d.source.y; })
        .attr("x2", function(d) { return d.target.x; })
        .attr("y2", function(d) { return d.target.y; });

      node.attr("cx", function(d) { return d.x; })
        .attr("cy", function(d) { return d.y; });
    });
  });

  setTimeout("loop();", 15 * 1000);
}

loop();
tick();
over();
window.onmousemove = move;

force.html - The main page that puts everything together

<html>
	<head>
		<title>Farm Visual</title>
		<script type="text/javascript" src="d3.v2.js"></script>
		<style>
			circle.node
			{
				stroke: #000;
				stroke-width: 2.0px;
			}
			
			line.link
			{
				stroke: #999;
				stroke-opacity: 0.75;
			}
			
			svg
			{
				border: 1px solid black;
			}
			
			#info, #list
			{
				white-space: nowrap;
			}
		</style>
		<link href="/scripts/bootstrap/css/bootstrap.css" rel="stylesheet">
		<link href="/scripts/bootstrap/css/bootstrap-responsive.css" rel="stylesheet">
		<script src="/scripts/jquery.js"></script>
		<script src="/scripts/bootstrap/js/bootstrap.js"></script>
	</head>
	
	<body>
		<center>
			<table width="100%">
				<tr><td colspan="2"><div id='info' class='alert alert-info' style='margin: 5px;'><b>Build Farm Status</b></div></td></tr>
				<tr><td><div class="gallery" id="chart" style="margin: 5px;"></div></td>
				<td valign="top" width="100%"><div id="list"></div></td></tr>
			</table>
		</center>
		<link href="force.css" rel="stylesheet" type="text/css" />
		<script src="force.js" type="text/javascript"></script>
	</body>
</html>

One thought on “Monitoring Our Build Farm — Visually”

  1. Howdy! Someone in my Facebook group shared this website with us
    so I came to give it a look. I’m definitely enjoying the information. I’m book-marking and will be
    tweeting this to my followers! Excellent blog and terrific design.

    Reply

Leave a comment