Wikipedia Monitoring IRC Bot

Sep 03 06:21:19 <wikimon> [ http://zenit.senecac.on.ca/wiki/index.php?title=CDOT& ] was changed by [ ShaneStrauss ] on [ 06:18, 3 September 2011 ]
Sep 03 09:11:19 <wikimon> [ http://zenit.senecac.on.ca/wiki/index.php?title=CDOT& ] was changed by [ Fossjon ] on [ 09:08, 3 September 2011 ]

#!/usr/bin/python

import os
import re
import select
import socket
import sys
import time
import urllib2

def listfilt(listobjc, filtstri, onceonly=0):
	outpflag = 0
	outplist = []
	
	for listitem in listobjc:
		if (re.match("^" + filtstri + "$", listitem)):
			outplist.append(listitem)
			
			if (onceonly == 1):
				outpflag = 1
		
		elif (outpflag == 1):
			outplist.append(listitem)
	
	return outplist

def getspage(urlnstri):
	headitem = ""
	
	try:
		websobjc = urllib2.urlopen(urlnstri, None, 10)
		dataread = websobjc.read().replace("\t", "").replace("\r", "").replace("\n", "")
		
		listlist = dataread.replace("<li", "\n<li").replace("</li", "</li\n").split("\n")
		listlist = listfilt(listlist, "<li.*checked.*")
		
		linklist = re.sub("(<a [^>]+>[^<]+</a>)", "\n\\1\n", listlist[0]).split("\n")
		linklist = listfilt(linklist, "<a [^>]+>[^<]+</a>.*")
		
		pagestri = re.sub("action=history[&]*", "", urlnstri).strip()
		datestri = re.sub("<[^>]+>", "", linklist[1]).strip()
		namestri = re.sub("<[^>]+>", "", linklist[2]).strip()
		
		headitem = ("[ %s ] was changed by [ %s ] on [ %s ]" % (pagestri, namestri, datestri))
	
	except KeyboardInterrupt:
		sys.exit(0)
	
	except:
		pass
	
	return headitem

sendline_last = 0; sendline_list = []

def sendline(circobjc, sendstri):
	global sendline_last
	global sendline_list
	
	prestime = time.time()
	tempstri = sendstri.strip()
	
	if (tempstri != ""):
		sendline_list.append(tempstri)
	
	if (((prestime - sendline_last) >= 1) and (len(sendline_list) > 0)):
		print("[SEND] %s" % (sendline_list[0]))
		circobjc.send(sendline_list[0] + "\r\n")
		
		sendline_list.pop(0)
		sendline_last = prestime

readline_data = ""

def readline(circobjc):
	global readline_data
	
	outpstri = ""
	(readlist, outplist, errolist) = select.select([circobjc], [], [], 0)
	
	if (circobjc in readlist):
		tempstri = circobjc.recv(2**20)
		
		if (not tempstri):
			readline_data = ""
			return None
		
		readline_data += tempstri
	
	try:
		newlindx = readline_data.index("\n")
	
	except:
		newlindx = -1
	
	if (newlindx != -1):
		outpstri = readline_data[:newlindx].strip()
		newlindx += 1
		readline_data = readline_data[newlindx:]
	
	return outpstri

procirco_last = 0

def procirco(circobjc, channame, lastmesg):
	global procirco_last
	
	prestime = time.time()
	
	if ((prestime - procirco_last) >= 60):
		sendline(circobjc, "JOIN " + channame)
		procirco_last = prestime
	
	regxobjc = re.match("^PING (.*)$", lastmesg)
	
	if (regxobjc):
		sendline(circobjc, "PONG " + regxobjc.group(1))

def newcirco(hostname, nickname):
	circobjc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	
	circobjc.connect((hostname, 6667))
	sendline(circobjc, "USER Alex * * : Bob")
	sendline(circobjc, "NICK %s" % (nickname))
	
	return circobjc

def main():
	if (len(sys.argv) < 4):
		print("Usage: %s <host> <nick> <chan>" % (sys.argv[0]))
		sys.exit(0)
	
	hostname = sys.argv[1]; nickname = sys.argv[2]; channame = sys.argv[3]
	pagelist = []; lastlist = []; lasttime = 0
	circobjc = None; circline = ""; circflag = 0
	
	while (1):
		circflag = 0
		prestime = time.time()
		
		if (circobjc == None):
			circobjc = newcirco(hostname, nickname)
		
		else:
			circline = readline(circobjc)
			
			if (circline == None):
				circobjc = None
			
			else:
				circflag = 1
				
				if (circline != ""):
					print("[RECV] %s" % (circline))
				
				procirco(circobjc, channame, circline)
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s add (.*)$" % (nickname), circline, re.I)
				
				if (regxobjc):
					if (regxobjc.group(2) not in pagelist):
						pagelist.append(regxobjc.group(2))
						lastlist.append("")
						lasttime = 0
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s del (.*)$" % (nickname), circline, re.I)
				
				if (regxobjc):
					for x in range(len(pagelist) - 1, -1, -1):
						try:
							if (re.match(regxobjc.group(2), pagelist[x], re.I)):
								sendline(circobjc, "PRIVMSG %s :deleting [ %s ]" % (regxobjc.group(1), pagelist[x]))
								lastlist.pop(x)
								pagelist.pop(x)
						
						except:
							pass
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s out.*$" % (nickname), circline, re.I)
				
				if (regxobjc):
					for listitem in lastlist:
						sendline(circobjc, "PRIVMSG %s :%s" % (regxobjc.group(1), listitem))
			
			if ((prestime - lasttime) >= (5 * 60)):
				for x in range(0, len(pagelist)):
					tempoutp = getspage(pagelist[x])
					
					if ((tempoutp != "") and (tempoutp != lastlist[x])):
						if (lastlist[x] != ""):
							sendline(circobjc, "PRIVMSG %s :%s" % (channame, tempoutp))
						
						lastlist[x] = tempoutp
				
				lasttime = prestime
		
		sendline(circobjc, "")
		
		if (circflag == 0):
			time.sleep(1)

if (__name__ == "__main__"):
	main()

Start Of The Semester

I just wanted to document a little more on what is pointing to what on our builder file systems. I put the commands and output below so hopefully one (or I) can follow it in the future in case file permissions or links get messed up.

# cat /etc/fstab

/dev/sdb1               /                       ext3    defaults             1 1
devpts                  /dev/pts                devpts  gid=5,mode=620       0 0
tmpfs                   /dev/shm                tmpfs   defaults             0 0
proc                    /proc                   proc    defaults             0 0
sysfs                   /sys                    sysfs   defaults             0 0
/hkexport/fs0.ext3      /fs0                    ext3    defaults,noauto,loop 0 0
/hkexport/swap0.ext3	/swap0                  ext3    defaults,noauto,loop 0 0
hongkong:/mnt/koji      /mnt/koji               nfs     defaults,noauto      0 0

# mount

/dev/sdb1 on / type ext3 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
australia:/var/export/cdot-guru-1-2 on /hkexport type nfs (rw,addr=192.168.1.196)
/dev/loop0 on /fs0 type ext3 (rw)
/dev/loop1 on /swap0 type ext3 (rw)
hongkong:/mnt/koji on /mnt/koji type nfs (rw,addr=192.168.1.254)

# ls -l /redirect

lrwxrwxrwx 1 root root 4 2010-11-05 13:34 /redirect -> /fs0

# for foldname in bin dev etc lib opt sys tmp usr var ; do find /${foldname} -type l -exec ls -l {} \; 2> /dev/null | grep -Ei ‘\-> /(fs0|redirect|hkexport|mnt/koji)’ ; done

lrwxrwxrwx 1 root root 23 2010-11-05 13:34 /var/lib/mock -> /redirect/mock/lib/mock
lrwxrwxrwx 1 root root 4 2010-11-05 13:34 /var/redirect -> /fs0
lrwxrwxrwx 1 root root 25 2010-11-05 13:34 /var/cache/mock -> /redirect/mock/cache/mock

Nearing The End Of The Summer

So below is a partial screen-shot of Styrene’s web interface. It is now using the updated processing.js to display a “click-able” bar graph and scrolling text-list of the current package states. I am posting this to show how much better the new lines and text fonts look now.

In addition, on Friday, I had followed Jon Masters guide on setting up the new ARMv7 panda boards and I had modified this script to help automate that. You just have to point it to the extracted paths of the uboot, boot and root folders and this script will auto partition the sd card for you. Here it is below:

#!/bin/bash
echo "Usage: $0 /dev/device /path/to/uboot /path/to/boot /path/to/root"
if [ ! "$1" = "/dev/sda" ] ; then
        DRIVE=$1
        for i in {1..5}
        do
        	umount ${DRIVE}${i} 2> /dev/null
        done
        if [ -b "$DRIVE" ] ; then
                dd if=/dev/zero of=$DRIVE bs=1024 count=1024
                SIZE=`fdisk -l $DRIVE | grep Disk | awk '{print $5}'`
                echo DISK SIZE - $SIZE bytes
                CYLINDERS=`echo $SIZE/255/63/512 | bc`
                echo CYLINDERS - $CYLINDERS
                {
                echo ,9,0x0C,*
                echo ,64,L,-
                echo ,64,S,-
                echo ,,,-
                } | sfdisk -D -H 255 -S 63 -C $CYLINDERS $DRIVE
                mkdosfs -F 16 -n uboot ${DRIVE}1
                mkfs.ext3 -L boot ${DRIVE}2
                mkswap ${DRIVE}3
                mkfs.ext3 -L root ${DRIVE}4
                umount /mnt/tmp 2> /dev/null
                rm -rf /mnt/tmp 2> /dev/null
                mkdir /mnt/tmp 2> /dev/null
                if [ "$2" != "" ]
                then
                	mount ${DRIVE}1 /mnt/tmp
                	cp ${2}/MLO /mnt/tmp/
                	sync
                	umount /mnt/tmp
                	mount ${DRIVE}1 /mnt/tmp
                	rsync -vr --exclude=MLO ${2}/ /mnt/tmp/
                	umount /mnt/tmp
                fi
                if [ "$3" != "" ]
                then
                	mount ${DRIVE}2 /mnt/tmp
                	cp -ax ${3}/* /mnt/tmp
                	umount /mnt/tmp
                fi
                if [ "$4" != "" ]
                then
                	mount ${DRIVE}4 /mnt/tmp
                	cp -ax ${4}/* /mnt/tmp
                	umount /mnt/tmp
                fi
                eject ${DRIVE}
        fi
fi


[secret] Modified Koji Hosts Page

So I tried to make the tiniest change possible to one of the pages on our build system and was faced with much up-roar about it. It was a small change that tried to allow our team to easily identify downed hosts in a quicker manner by highlighting them. The other members of the community didn’t seem to appreciate me changing the source code of the system (even tho it was a small change to only one page!) so they practically made me change it back. They don’t even have access to our build system so the change was really to make our lives a bit easier and it was a page that they weren’t even required to visit. Anyway, I’m just going to leave this little easter-egg here for my team lol! I’m at a loss for words so wow is all I can say…

Edit: If they find out about this “feature” I might need some tips on obfuscating javascript code with something that encrypts, encodes, decodes, decrypts, evals and executes js on the fly, you know, like what the “bad” guys do!

Edit2: Just some thoughts on what I could use, the worst case is that the JS code is fully CIA’d with a common shared secret key that only our team knows and the easter-egg is something un-guessable. That would be a pain to use but there’s pretty much no way (as of today) anyone could crack that given a strong enough password lol 🙂

Confidentiality: AES-256-CBC
Integrity: SHA512
Authentication: HMAC_SHA512

<script>
function montostr(monthstr)
{
	var monthlst = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
	var x;
	monthstr = monthstr.substring(0, 3).toLowerCase();
	for (x = 0; x < monthlst.length; ++x)
	{
		if (monthstr == monthlst[x])
		{
			return x;
		}
	}
	return 0;
}
function magiclik()
{
	document.body.style.background = "#000000";
	var dateobjc = document.getElementById("loginInfo");
	var datestri = dateobjc.innerHTML.replace(/[\t\r\n]/g, "").replace(/^[^0-9A-Za-z]+/, "").replace(/UTC.*$/, "").replace(/[^0-9A-Za-z]+$/, "");
	var datelist = datestri.replace(/ /g, ":").split(":");
	datelist[2] = montostr(datelist[2]);
	var datetime = new Date(datelist[3], datelist[2], datelist[1], datelist[4], datelist[5], datelist[6]);
	var rowdlist = document.getElementsByTagName("tr");
	var x, coldlist, hostdati;
	for (x = 0; x < rowdlist.length; ++x)
	{
		if (!rowdlist[x].innerHTML.replace(/[\t\r\n]/g, "").match(/^.*<a href="hostinfo\?hostID=[0-9]+">[^<]+<\/a>.*$/))
		{
			continue;
		}
		coldlist = rowdlist[x].getElementsByTagName("td");
		datelist = coldlist[5].innerHTML.replace(/[\t\r\n]/g, "").replace(/[^0-9]/g, ":").split(":");
		if (datelist.length < 6)
		{
			continue;
		}
		datelist[1] = (parseInt(datelist[1].replace(/^[^1-9]/g, "")) - 1);
		hostdati = new Date(datelist[0], datelist[1], datelist[2], datelist[3], datelist[4], datelist[5]);
		coldlist = coldlist[1].getElementsByTagName("a");
		if ((hostdati.getTime() / 1000) <= ((datetime.getTime() / 1000) - (15 * 60)))
		{
			coldlist[0].innerHTML = ("<font color='red'>" + coldlist[0].innerHTML + "</font>");
		}
		else
		{
			coldlist[0].innerHTML = ("<font color='green'>" + coldlist[0].innerHTML + "</font>");
		}
	}
}
function magicmod()
{
	var imagobjc = document.getElementById("kojiLogo");
	imagobjc.onclick = magiclik;
}
if (location.href.match(/^.*hosts.*$/))
{
	window.onload = magicmod;
}
</script>

[update] Koji Hosts Page

There is a bug in the code-change I wrote to modify the look of the Koji Build System Hosts page. If the last-update timestamp was blank, it would cause the embedded py script to crash and freeze the page. Below is how the “better” code should read…

Before:

After:

      #for $host in $hosts
        <tr class="$util.rowToggle($self)">
          <td>$host.id</td>
        #if ($util.formatTime($host.last_update) != ""):
          #if (($time.time() - $time.mktime($time.strptime($util.formatTime($host.last_update),'%Y-%m-%d %H:%M:%S'))) < (15 * 60)):
            <td><a href="hostinfo?hostID=$host.id"><font color="green">$host.name</font></a></td>
          #else
            <td><a href="hostinfo?hostID=$host.id"><font color="red">$host.name</font></a></td>
          #end if
        #else
          <td><a href="hostinfo?hostID=$host.id">$host.name</a></td>
        #end if
          <td>$host.arches</td>
          <td class="$str($bool($host.enabled)).lower()">#if $host.enabled then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td class="$str($bool($host.ready)).lower()">#if $host.ready then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td>$util.formatTime($host.last_update)</td>
        </tr>
      #end for

The Web Part Of Styrene

I was requested to blog more about my actual work instead of the huge flood of off-topic posts that I usually do. So here is a post about Styrene’s web interface end of things. As you may see, one is able to perform a range of package/build “actions” and Styrene is able to handle multiple users as well.



How to use the command line:
– Make a config file like the example provided below
– Load a DB with the required package set and dependencies
– Mark the DB initially to get the latest build information
– Run the koji-que loop in one bash shell to auto build packages
– Run the mark-db every 1+ hours to refresh the DB information

Here is how the back-end of things look:

Usage: styrene <command>	[--db=<db name>] [--release=<release number>] [--kojihub=<koji url>] [--kojitag=<tag name>]
				[--binrepo=<bin repo>] [--srcrepo=<src repo>] [--srcarch=<src arch>]
				[--pkg=<pkg name>] [--state=<pkg state>] [--comment=<pkg comment>] [--override=<envr override>]
				[--num=<general number>] [--time=<time in seconds>]

Commands:

(Db Operations)
			load-db			Create and load a local database with packages from a specific architecture
			update-db		Append a local database with packages from a specific architecture

(Pkg Info)
			list-unbuilt		Display a list of currently unbuilt packages
			list-ready		List any packages that are ready to be built (all dependencies satisfied)
			pkg-info		Display various package info/stats from the local db
			mark-pkg		Mark a pkg state in the local db

(Koji Info)
			koji-info		Get various info/stats for a pkg from koji
			koji-prev		Print the latest built version of a package if one exists
			koji-equiv		Check to see if koji has a pkg version that is >= to this local db version
			koji-que		Send a package to koji for a build attempt
			koji-tasks		List the current active tasks for this user

(Auto Building)
			mark-db			foreach (list-unbuilt) use (koji-info) to (mark-pkg)
			que-ready		foreach (list-ready) use (koji-tasks) to throttle (koji-que)

(Web Interface)
			cgi-bin			Generate a CSS/HTML/JavaScript (web-based) interface

(Pkg Fixing)
			fix-spec		Attempt to auto-fix a non-building SRPM by auto-merging old spec file changes

(Misc Cmds)
			show-vars		Print the different variable values for info purposes
			show-deps		Look up the related dependencies for a given pkg
			build-model		Run a continuous loop to simulate a build order until the pkgs aren't satisfied (modifies the model field)

(Example)
			~/.styrene.conf		db = /tmp/ps.db
						kojitag = dist-f13-updates
						release = 13
						binrepo = updates
						srcrepo = updates-source

[random] 2x Off-Topic Ideas

So Google employee’s are required to take a break from work and experiment with anything they’re interested in (projects like gmail arose from this!). That was just a random side-note which may or may not be related to what’s below. This will probably be one of my more random posts on this blog.

RSA Animate – Drive: The surprising truth about what motivates us
(Full Version)

Anyway, here’s some tmp stuff I don’t really need but I guess this can be considered a backup, who knows…

rm -fv /tmp/f15.*.list
(curl -s 'http://git.fedorahosted.org/git/?p=arm.git;a=tree;f=stage3/armv7hl/SRPMS;hb=refs/heads/armv7hl' | tr -d '\t\r\n' | awk '{ gsub(/<tr/, "\n<tr"); gsub(/<\/tr/, "\n"); print; }' | grep -i '<td class="mode">' | sed -e 's/<[^>]*>/ /g' | awk '{ print $3 }') | tee /tmp/f15.git.src.list
(curl -s 'http://ausil.us/fedora-arm/mock/' | sed -e 's/<[^>]*>/ /g' | awk '{ print $1 }' | grep -i '.*/$' | while read line ; do test=`curl -s "http://ausil.us/fedora-arm/mock/$line" | sed -e 's/<[^>]*>/ /g' | awk '{ print $1 }' | grep -i '\.src\.rpm$'` ; if [ "$test" != "" ] ; then echo "$test" ; fi ; done) | tee /tmp/f15.dg.src.list
(curl -s 'http://ausil.us/fedora-arm/SRPMS/' | sed -e 's/<[^>]*>/ /g' | awk '{ print $1 }' | grep -i '\.src.rpm$') | tee /tmp/f15.dg.alt.list
(curl -s 'http://ausil.us/fedora-arm/noarch/' | sed -e 's/<[^>]*>/ /g' | awk '{ print $1 }' | grep -i '\.noarch.rpm$') | tee /tmp/f15.dg.noar.list
gunzip < /home/jchiappetta/Desktop/arm/styrene/misc-files/f15.db.gz > /tmp/f15.db
(cat /tmp/f15.*.list | sed -e 's/^\([^\.]*-[0-9]*\.[0-9]*\)[0-9\.]*.*$/\1*/g' | ./styrene mark-pkg --state=1) | tee /tmp/f15.mark.log

<?php
	function webs($urls)
	{
		$fobj = fopen($urls, "r");
		$page = "";
		while ($fobj && !feof($fobj))
		{
			$page .= fgets($fobj, 4096);
		}
		fclose($fobj);
		
		$page = str_replace("\t", "", $page);
		$page = str_replace("\r", "", $page);
		$page = str_replace("\n", "", $page);
		$page = preg_replace("/>[ ]*</", "><", $page);
		
		return $page;
	}
	
	$urls = $_GET["page"];
	$pref = preg_replace("/^http:\/\//", "", $urls);
	$urls = ("http://".$pref);
	$root = ("http://".preg_replace("/\/.*$/", "", $pref));
	
	$dsts = webs($urls);
	$dsts = preg_replace("/href=(['\"])\//", "href=\\1".$root."/", $dsts);
	$dsts = str_replace("</li></ul></td></tr></table>", "</li></ul></td></tr></table>\n", $dsts);
	
	$dstl = explode("\n", $dsts);
	
	if (count($dstl) > 1)
	{
		$rsss = webs("http://reddit.com/.rss");
		$rsss = str_replace("<item", "\n<item", $rsss);
		$rsss = str_replace("</item", "</item\n", $rsss);
		
		$rssl = explode("\n", $rsss);
		
		foreach ($rssl as $line)
		{
			if (preg_match("/^<item.*$/", $line))
			{
				$head = $line;
				$head = preg_replace("/^.*<title>/", "", $head);
				$head = preg_replace("/<\/title>.*$/", "", $head);
				
				$desc = $line;
				$desc = preg_replace("/^.*<description>/", "", $desc);
				$desc = preg_replace("/<\/description>.*$/", "", $desc);
				
				$dstl[0] .= ("<h2><span class=\"mw-headline\">".$head."</span></h2>");
				
				$desc = str_replace("&lt;", "<", $desc);
				$desc = str_replace("&gt;", ">", $desc);
				$desc = str_replace("&quot;", "\"", $desc);
				
				$dstl[0] .= $desc;
			}
		}
		
		echo $dstl[0].$dstl[1]."\n";
	}
?>

[final] Patching The Koji Hosts Page

So you edit this file

/usr/share/koji-web/scripts/hosts.chtml

change the displaying if-block

    #if $len($hosts) > 0
      #for $host in $hosts
        #if (($time.time() - $time.mktime($time.strptime($util.formatTime($host.last_update),'%Y-%m-%d %H:%M:%S'))) < (15 * 60)):
          #continue
        #end if
        <tr class="$util.rowToggle($self)">
          <td>$host.id</td>
          <td><a href="hostinfo?hostID=$host.id"><b><font color="red">$host.name</font></b></a></td>
          <td>$host.arches</td>
          <td class="$str($bool($host.enabled)).lower()">#if $host.enabled then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td class="$str($bool($host.ready)).lower()">#if $host.ready then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td>$util.formatTime($host.last_update)</td>
        </tr>
      #end for
      #for $host in $hosts
        #if (($time.time() - $time.mktime($time.strptime($util.formatTime($host.last_update),'%Y-%m-%d %H:%M:%S'))) >= (15 * 60)):
          #continue
        #end if
        <tr class="$util.rowToggle($self)">
          <td>$host.id</td>
          <td><a href="hostinfo?hostID=$host.id"><b><font color="green">$host.name</font></b></a></td>
          <td>$host.arches</td>
          <td class="$str($bool($host.enabled)).lower()">#if $host.enabled then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td class="$str($bool($host.ready)).lower()">#if $host.ready then $util.imageTag('yes') else $util.imageTag('no')#</td>
          <td>$util.formatTime($host.last_update)</td>
        </tr>
      #end for
    #else

– OR THE HARDER WAY –

edit

/usr/share/koji-web/scripts/includes/header.chtml

add this line right before the ending </head> tag

#include "includes/hosts.js"

and you create a file in the same directory named hosts.js with

<script>
	var hoststat = 0;
	
	function conmonth(monthstr)
	{
		var x, l, m;
		var monthary = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
		
		l = monthary.length;
		m = monthstr.length;
		
		for (x = 0; x < l; ++x)
		{
			if (monthary[x].substring(0, m) == monthstr)
			{
				return (x + "");
			}
		}
		
		return (0 + "");
	}
	
	function procdown(rowdlist)
	{
		var x, l = rowdlist.length, z;
		var coldlist, objclist;
		var locldate, loclutct;
		var hostdate, hostutct;
		var logiinfo = document.getElementById("loginInfo");
		var dateobjc = logiinfo.innerHTML.replace(/[\t\r\n]/g, "").match(/^.*[^ ]+ ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) UTC.*$/);
		
		locldate = (dateobjc[3] + ":" + conmonth(dateobjc[2]) + ":" + dateobjc[1] + ":" + dateobjc[4]).split(":");
		loclutct = new Date(locldate[0], locldate[1], locldate[2], locldate[3], locldate[4], locldate[5]);
		
		for (x = 0; x < l; ++x)
		{
			coldlist = rowdlist[x].getElementsByTagName("td");
			
			if (coldlist.length < 6)
			{
				continue;
			}
			
			hostdate = coldlist[5].innerHTML.replace(/^[^0-9]*/g, "").replace(/[^0-9]*$/g, "").replace(/[^0-9]/g, ":").split(":");
			
			if (hostdate.length < 6)
			{
				continue;
			}
			
			objclist = coldlist[1].getElementsByTagName("a");
			hostutct = new Date(hostdate[0], parseInt(hostdate[1].replace(/^[^1-9]*/g, "")) - 1, hostdate[2], hostdate[3], hostdate[4], hostdate[5]);
			
			if (objclist.length > 0)
			{
				if ((hostutct.getTime() / 1000) <= ((loclutct.getTime() / 1000) - (15 * 60)))
				{
					objclist[0].innerHTML = ("<b><font color=\"red\">" + objclist[0].innerHTML + "</font></b>");
				}
				
				else
				{
					objclist[0].innerHTML = ("<b><font color=\"green\">" + objclist[0].innerHTML + "</font></b>");
				}
			}
		}
	}
	
	function redotabl()
	{
		var x, l;
		var downlist = [], rowdlist = ["row-odd", "row-even"];
		var tabllist = document.getElementsByClassName("data-list");
		var tablrowd;
		
		if (tabllist.length > 0)
		{
			l = tabllist[0].rows.length;
			
			for (x = 0; x < l; ++x)
			{
				if (tabllist[0].rows[x].innerHTML.replace(/[\t\r\n]/g, "").match(/^.*font.*color.*red.*$/))
				{
					downlist.unshift([x, tabllist[0].rows[x].innerHTML]);
				}
			}
			
			l = downlist.length;
			
			for (x = 0; x < l; ++x)
			{
				tabllist[0].deleteRow(downlist[x][0]);
			}
			
			for (x = 0; x < l; ++x)
			{
				tablrowd = tabllist[0].insertRow(3);
				tablrowd.innerHTML = downlist[x][1];
			}
			
			l = tabllist[0].rows.length;
			
			for (x = 3; x < l; ++x)
			{
				tabllist[0].rows[x].className = rowdlist[(x - 3) % 2];
			}
		}
	}
	
	function hostloop()
	{
		var oddrlist = document.getElementsByClassName("row-odd");
		var evenlist = document.getElementsByClassName("row-even");
		
		if ((oddrlist.length > 0) || (evenlist.length > 0))
		{
			procdown(oddrlist);
			procdown(evenlist);
			redotabl();
			
			hoststat = 1;
		}
		
		if (hoststat == 0)
		{
			setTimeout("hostloop();", 0.5 * 1000);
		}
	}
	
	if (location.href.match(/^.*hosts.*$/))
	{
		hostloop();
	}
</script>

[arm-koji] Modified Hosts Page In PHP

So the other day, I published some JS code to change the look of the ARM-Koji Hosts page. It assumed that each client’s time was set to Eastern time zone settings (really very bad practice!). Since I have access to a PHP-based Apache server, I wanted to see how many lines it took me to implement the same.

<?php
	# get the koji host-status web page
	$u = "http://arm.koji.fedoraproject.org/koji/hosts";
	$hostpage = file_get_contents($u);
	$hostpage = str_replace("\t", "", $hostpage);
	$hostpage = str_replace("\r", "", $hostpage);
	$hostpage = str_replace("\n", "", $hostpage);
	# prep the page to be processed for host names and times
	$tabldata = $hostpage;
	$tabldata = str_replace("<tr", "\n<tr", $tabldata);
	$tabldata = str_replace("</tr>", "</tr>\n", $tabldata);
	$tabldata = explode("\n", $tabldata);
	# store the host names and times in a new list
	$hostlist = array();
	$begindex = -1;
	for ($x = 0; $x < count($tabldata); $x += 1)
	{
		if (preg_match("/^.*<a href=\"hostinfo\?hostID=([0-9]+)\">([^<]+)<\/a>.*>([0-9]+-[0-9]+-[0-9]+ [0-9]+:[0-9]+:[0-9]+).*$/", $tabldata[$x], $machlist))
		{
			$hosttime = preg_replace("/[^0-9]/", "", $machlist[3]);
			$templist = array(0, $machlist[1], $machlist[2], $machlist[3], $hosttime);
			array_push($hostlist, $templist);
		}
		if (preg_match("/^.*<a href=\"hostinfo\?hostID=[0-9]+\">[^<]+<\/a>.*$/", $tabldata[$x]))
		{
			$begindex = $x;
			$tabldata[$x] = "";
		}
	}
	# loop thru the list and mark/sort the ones that up/down
	$timesecs = time();
	$utcsdate = gmdate("YmdHis", $timesecs);
	$utcstime = (intval($utcsdate) - (15 * 60));
	for ($x = 0; $x < count($hostlist); $x += 1)
	{
		if (intval($hostlist[$x][4]) <= $utcstime)
		{
			$hostlist[$x][0] = 0;
			$hostlist[$x][2] = ("<b><font color='red'>".$hostlist[$x][2]."</font></b>");
		}
		else
		{
			$hostlist[$x][0] = 1;
			$hostlist[$x][2] = ("<b><font color='green'>".$hostlist[$x][2]."</font></b>");
		}
	}
	sort($hostlist);
	$headlist = array("row-odd", "row-even");
	for ($x = 0; $x < count($hostlist); $x += 1)
	{
		$c = $headlist[$x % 2];
		$i = $hostlist[$x][1];
		$n = $hostlist[$x][2];
		$t = $hostlist[$x][3];
		$tabldata[$begindex] .= ("<tr class='".$c."'><td>".$i."</td><td><a href='".$u."/hostinfo?hostID=".$i."'>".$n."</a></td><td>.</td><td>.</td><td>.</td><td>".$t."</td></tr>\n");
	}
	echo (join("\n", $tabldata)."\n");
?>


[quick] Preparing For The F15 Build Process

So before we can run styrene for the F15 build process, we first must load/initialize it with a seed package set. This package set has to be manually built by the Fedora-ARM members. Since we are storing this package set in git, I needed a way to retrieve a list of what is currently built and load it into styrene. Here is the command below I used to do so:

curl -s 'http://git.fedorahosted.org/git/?p=arm.git;a=tree;f=stage3/armv7hl/SRPMS;hb=refs/heads/armv7hl' | tr -d '\t\r\n' | awk '{ gsub(/<tr/, "\n<tr"); gsub(/<\/tr/, "\n"); print; }' | grep -i '<td class="mode">' | sed -e 's/<[^>]*>/ /g' | awk '{ print $3 }'

Hightlighting Downed Hosts In JS

So if you visit

http://arm.koji.fedoraproject.org/koji/hosts

And you enter this into your URL bar

javascript:
function custom_pad(n)
{
	if (n < 10)
	{
		return ("0" + n);
	}
	return ("" + n);
}
function custom_chg(k, l)
{
	var x, indxlist = [], rowdlist = [];
	for (x = 0; x < l.length; ++x)
	{
		var t = new Date();
		var s = parseInt(t.getTime() / 1000);
		var r = new Date((s + (4 * 60 * 60) - (30 * 60)) * 1000);
		var q = (r.getFullYear()+""+custom_pad(r.getMonth() + 1)+""+custom_pad(r.getDate())+""+custom_pad(r.getHours())+""+custom_pad(r.getMinutes())+""+custom_pad(r.getSeconds()));
		var c = l[x].getElementsByTagName("td");
		if (c.length < 6)
		{
			continue;
		}
		var d = c[5].innerHTML.replace(/[^0-9]/g, "");
		var u = "green";
		if (parseInt(d) <= parseInt(q))
		{
			u = "red";
		}
		var o = c[1].getElementsByTagName("a");
		for (var y = 0; y < o.length; ++y)
		{
			var n = o[y].innerHTML.replace(/<[^>]+>/g, "");
			o[y].innerHTML = ("<b><font color='" + u + "'>" + n + "</font></b>");
			o[y].href = ("http://arm.koji.fedoraproject.org/koji/" + o[y].href.replace(/^.*\//g, ""));
		}
		if (u == "red")
		{
			indxlist.push(x);
			rowdlist.push(l[x].innerHTML);
		}
	}
	for (x = (indxlist.length - 1); x > -1; --x)
	{
		k.deleteRow(indxlist[x]);
	}
	for (x = (rowdlist.length - 1); x > -1; --x)
	{
		var v = k.insertRow(1);
		v.innerHTML = rowdlist[x];
	}
	var rowdtype = ["row-odd", "row-even"];
	for (x = 1; x < l.length; ++x)
	{
		l[x].className = rowdtype[(x - 1) % 2];
	}
}
if (location.href.match(/^.*(fedoraproject.org|hongkong).*hosts.*/))
{
	var h, i, j;
	h = null;
	while (!h)
	{
		h = document.getElementById("header");
	}
	h.parentNode.removeChild(h);
	h = null;
	while (!h)
	{
		h = document.getElementById("mainNav");
	}
	h.parentNode.removeChild(h);
	h = null;
	while (!h)
	{
		h = document.getElementById("loginInfo");
	}
	h.parentNode.removeChild(h);
	i = null;
	while (!i)
	{
		i = document.getElementsByTagName("h4");
	}
	i[0].parentNode.removeChild(i[0]);
	i = null; j = null;
	while (!i && !j)
	{
		i = document.getElementsByClassName("data-list");
		j = document.getElementsByTagName("tr");
	}
	i[0].deleteRow(0);i[0].deleteRow(0);
	custom_chg(i[0], j);
}

You get a coloured list of downed hosts!

Dual File Compare in py

#!/usr/bin/python
import os
import sys

filelist = sys.argv[1:]
fileleng = len(filelist)

maxlnumb = 0
linelist = []

for filename in filelist:
	fobjitem = file(filename)
	fobjdata = fobjitem.readlines()
	fobjitem.close()
	
	linenumb = len(str(len(fobjdata)))
	maxlnumb = max(maxlnumb, linenumb)
	linelist.append(fobjdata)

spacereq = (maxlnumb + 1 + 4)
maxsline = ((int(os.environ["COLUMNS"]) - (spacereq * fileleng)) / fileleng)
x = 1

while (1):
	quitflag = 1
	linenumb = str(x)
	outpstri = ""
	
	while (len(linenumb) < maxlnumb):
		linenumb = ("0" + linenumb)
	
	for lineitem in linelist:
		linestri = ""
		
		if (len(lineitem) > 0):
			linestri = lineitem.pop(0)
			quitflag = 0
		
		linestri = linestri.replace("\t", "    ")
		linestri = linestri[0:maxsline].rstrip()
		
		while (len(linestri) < maxsline):
			linestri += " "
		
		if (outpstri != ""):
			outpstri += " | "
		
		outpstri += (linenumb + ":" + linestri)
	
	if (quitflag == 1):
		break
	
	print(outpstri)
	x += 1

[arm-team] Freeing Possibly Stuck Tasks

Since I’ve finally been experimenting with mass queue builds, there have been tasks and builders which have become “stuck” over a long period of time.

To get your latest tasks:

arm-koji list-tasks --mine | grep -i '.*open.*noarch.* build .*' | awk '{ print $1 }' > tasks.log

To free tasks from their builders that were created more than 2 days ago:

cat tasks.log | while read tasknum ; do taskday=`arm-koji taskinfo "$tasknum" | grep -i '^created' | awk '{ print $4 }'` ; today=`date "+%d"` ; let today="$today - 2" ; if [ $taskday -le $today ] ; then echo "beg [$tasknum]" ; arm-koji free-task "$tasknum" ; echo "end [$tasknum]" ; fi ; sleep 1 ; done

Finally Getting To Really Test Styrene!

So after much scripting, I am finally getting to test and see if this thing actually works. It is currently attempting to automate the building of 2000+ Fedora 13 Update packages. I am completely aware and fully expecting there to be many packages that fail requiring manual, user intervention. I am hoping tho, however, that this tool helps to clear a bulk of the packages out of the way for our team. In addition, I will continue working on the visual component of it hopefully allowing our team to manually edit and customize builds. Maybe in the future this tool can attempt to automatically learn what specific fixes were applied to past packages and auto-apply them to future ones, who knows. Anyway, if you want to track the builds of any of our team members, you can run the following bash script below.

$ ./bcomp.sh fossjon 20110628

name:[liferea-1.6.5-1.fc13] date:[20110628]
name:[gwibber-3.0.0.1-2.fc13] date:[20110628]
name:[doxygen-1.7.3-1.fc13] date:[20110628]
name:[dovecot-1.2.17-1.fc13] date:[20110628]
name:[cups-1.4.6-1.fc13] date:[20110628]
name:[bash-completion-1.2-4.fc13] date:[20110628]
name:[NetworkManager-0.8.4-1.fc13] date:[20110628]
name:[dhcp-4.1.2-4.ESV.R2.fc13] date:[20110628]
...


#!/bin/bash
if [ "$1" == "" ]
then
	echo "Usage: $0 <account name> [start date]"
	exit 0
fi
x=0
s=0
if [ "$2" != "" ]
then
	s="$2"
fi
while true
do
	o=`curl -s "http://arm.koji.fedoraproject.org/koji/builds?userID=$1&order=-build_id&state=1&start=${x}" | tr -d '\t\r\n' | awk '{ gsub(/<tr/, "\n<tr"); print; }' | grep -i 'buildinfo?buildID=[0-9]*'`
	if [ "$o" == "" ]
	then
		break
	fi
	echo "$o" | while read l
	do
		p=`echo "$l" | sed -e 's@.*<td><a href="buildinfo?buildID=[0-9][0-9]*">\([^<][^<]*\)</a></td>.*@\1@g'`
		d=`echo "$l" | sed -e 's@.*<td>\([0-9][0-9][0-9][0-9]\)-\([0-9][0-9]\)-\([0-9][0-9]\) [0-9][0-9]:[0-9][0-9]:[0-9][0-9]</td>.*@\1\2\3@g'`
		q="name:[$p] date:[$d]"
		if [ $d -ge $s ]
		then
			echo "$q"
		fi
	done
	let x="$x + 50"
done