[updated] F15 Koji Que Script

Here it is, hopefully it works a bit better!

#!/bin/bash
export x=1
export l=`find ./f15srpms -type f | grep -i '\.src\.rpm' | wc -l`
koji list-tagged dist-f15 > /tmp/pkoji.txt
find ./f15srpms -type f | sort -R | grep -i '\.src\.rpm' > /tmp/pkgsl.txt
while true
do
	while true
	do
		ak list-tagged dist-f15 > /tmp/akoji.txt
		ak list-tasks --mine --quiet | grep '^[0-9]' | grep -Ei ' (open|free) .* build' > /tmp/tasks.txt
		#echo "number of our tasks == `cat /tmp/tasks.txt | wc -l`"
		n=`cat /tmp/tasks.txt | wc -l`
		if [ $n -ge 10 ]
		then
			break
		fi
		p=`cat /tmp/pkgsl.txt | head -n "$x" | tail -n 1`
		q=`basename "$p" | sed -e 's/-[^-]*-[^-]*$//'`
		let x="($x % $l) + 1"
		#echo "checking pkg [$p] name [$q] not built on (akoji) and built on (pkoji)"
		c=`cat /tmp/akoji.txt /tmp/tasks.txt | grep -i "$q"`
		if [ "$c" != "" ]
		then
			continue
		fi
		c=`cat /tmp/pkoji.txt | grep -i "$q"`
		if [ "$c" == "" ]
		then
			continue
		fi
		echo "queing [$p] position [$x]"
		ak build dist-f15 "$p" --nowait --background
	done
	sleep 60
done

2011 in review

The WordPress.com stats helper monkeys prepared a 2011 annual report for this blog.

Here’s an excerpt:

A San Francisco cable car holds 60 people. This blog was viewed about 1,400 times in 2011. If it were a cable car, it would take about 23 trips to carry that many people.

Click here to see the complete report.

[off-topic] Fighting Governments — DNS Re-Implemented

The U.S. government is taking down websites using DNS poisoning/hijacking methods on pages that are claimed to be infringing on copyright law. I have re-written a *really* basic DNS server which re-implements DNS forwarding/proxying, DNS caching and DNS un-blacklisting of A/IN/IPv4 records. Basically, if one were to run their own DNS server (like the one below) and maintain a simple list of government-poisoned IP addresses for it, the DNS server would be able to provide (previously expired but still good) IPv4 addresses for the blocked site. This could essentially allow one to re-access a now blacklisted website.

Here is a screen shot of my computer using this basic DNS server to visit slashdot:

$ host slashdot.org
slashdot.org has address 216.34.181.45
slashdot.org mail is handled by 10 mx.sourceforge.net.

And here is what it would look like if it were taken down via DNS poisoning:

$ host slashdot.org
slashdot.org has address 74.81.170.110
slashdot.org has address 216.34.181.45
slashdot.org mail is handled by 10 mx.sourceforge.net.

Once this happens, all one has to do now is append the newly offending IP address to the DNS server’s black list text file like so:

# cat dnsb.txt 
74.81.170.110

And if you perform another DNS request to the same (custom) DNS server, you get the last cached answers bringing you back to slashdot:

$ host slashdot.org
slashdot.org has address 216.34.181.45
slashdot.org mail is handled by 10 mx.sourceforge.net.

The code below is simply proof-of-concept software and should *NOT* be used for production as I have implemented everything from scratch here and know very little about how DNS really works (i.e. message compression).

import os
import re
import socket
import sqlite3
import sys
import time

sqlo = sqlite3.connect("dnsd.db")
sqlc = sqlo.cursor()

def fixs(inpt):
	outp = ""
	
	for letr in inpt:
		if (ord(letr) < 32):
			outp += "."
		
		else:
			outp += letr
	
	return outp

def rslv():
	nlst = []
	fobj = open("/etc/resolv.conf", "r")
	
	dnsl = fobj.readlines()
	
	for dnse in dnsl:
		item = dnse.strip("\0\t\r\n ")
		regx = re.match("^nameserver[ \t]*(.*)$", item)
		
		if (regx):
			addr = regx.group(1)
			nlst.append(addr)
	
	return nlst

def forw(addr, data):
	resp = ""
	serv = (addr, 53)
	nobj = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	
	try:
		nobj.settimeout(5)
		nobj.sendto(data, serv)
		resp = nobj.recv(2**20)
	
	except:
		pass
	
	try:
		nobj.close()
	
	except:
		pass
	
	return resp

def answ(data, deet=False):
	# http://www.ietf.org/rfc/rfc1035.txt
	
	requ = 13
	null = (requ + data[requ:].find("\0"))
	dtyp = (null + 5)
	dlen = (dtyp + 10)
	
	size = len(data)
	name = fixs(data[requ:null])
	begi = -1
	endi = -1
	ansl = []
	
	while ((dlen + 1) < size):
		dsiz = ((ord(data[dlen]) << 8) + ord(data[dlen + 1]))
		dsig = (dtyp + 4)
		
		if (((dlen + 5) < size) and (data[dtyp:dsig] == "\xc0\x0c\x00\x01")):
			if (deet == True):
				if (begi == -1):
					begi = dtyp
				
				else:
					endi = (dlen + 1 + dsiz + 1)
			
			addr = ""
			
			for x in range(2, 2 + 4):
				addr += ("." + str(ord(data[dlen + x])))
			
			ansl.append([name, addr[1:]])
		
		dtyp = (dlen + 1 + dsiz + 1)
		dlen = (dtyp + 10)
	
	if (begi == -1):
		begi = (null + 5)
		endi = begi
	
	if (deet == False):
		return ansl
	
	else:
		return [begi, endi]

def sqle(comd, outp=False):
	global sqlo
	global sqlc
	
	try:
		sqlc.execute(comd)
	
	except:
		pass
	
	if (outp == False):
		sqlo.commit()
	
	else:
		return sqlc.fetchall()

def filt():
	try:
		fobj = open("dnsb.txt", "r")
		
		for line in fobj.readlines():
			addr = line.strip("\0\t\r\n ")
			sqle("DELETE FROM data WHERE addr = '%s';" % (addr))
		
		fobj.close()
	
	except:
		return 0
	
	return 1

def cche(name):
	dbdl = sqle("SELECT * FROM data WHERE name = '%s';" % (name), outp=True)
	outp = []
	
	for dbdi in dbdl:
		adrl = dbdi[1].split(".")
		adrs = ""
		
		for adri in adrl:
			adrs += chr(int(adri))
		
		outp.append([int(dbdi[2]), adrs])
	
	outp.sort()
	outp.reverse()
	
	return outp

def shim(data, adrl):
	hedn = [6, 8]
	payn = answ(data, deet=True)
	
	lead = (payn[0] + 12)
	size = len(adrl)
	
	dlen = (chr((size >> 8) & 0xff) + chr((size >> 0) & 0xff))
	head = (data[:hedn[0]] + dlen + data[hedn[1]:payn[0]])
	
	payl = ""
	
	for adri in adrl:
		payl += (data[payn[0]:lead] + adri[1])
	
	payl += data[payn[1]:]
	
	outp = (head + payl)
	
	# beg: compression offset bypass
	
	outp = (outp[:8] + chr(0) + chr(0) + outp[10:])
	outp = (outp[:10] + chr(0) + chr(0) + outp[12:])
	
	# end: compression offset bypass
	
	# beg: compression offset fix
	'''
	x = 0
	l = len(outp)
	s = ((ord(data[6]) << 8) + ord(data[7]))
	d = ((size - s) * 16)
	
	if (d > 0):
		while ((x + 1) < l):
			o = ((ord(outp[x]) << 8) + ord(outp[x + 1]))
			
			if ((o & 0xff00) == 0xc000):
				o = (o & 0x00ff)
				
				if (o > 0x0c):
					o = (o + d)
					outp = (outp[:x] + chr((o >> 8) & 0xff) + chr((o >> 0) & 0xff) + outp[x + 2:])
			
			x = (x + 2)
	'''
	# end: compression offset fix
	
	return outp

def debg(addr, requ, answ):
	adrl = addr.split(".")
	adrl[3] = "x"
	prio = ""
	
	for letr in answ:
		prio += ("." + str(ord(letr)))
	
	print("[%s] requ [%s] answ [%s, ...]" % (".".join(adrl), requ, prio[1:]))

def serv():
	snam = ("", 53)
	sobj = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	
	sobj.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
	sobj.bind(snam)
	
	while (1):
		(data, addr) = sobj.recvfrom(2**20)
		rlvl = rslv()
		
		for rlvi in rlvl:
			dnsr = forw(rlvi, data)
			
			if (dnsr):
				resl = answ(dnsr)
				
				if (len(resl) > 0):
					for resi in resl:
						dnsq = re.sub("[^0-9A-Za-z\.\-]", "", resi[0])
						dnsa = re.sub("[^0-9\.]", "", resi[1])
						epch = str(int(time.time()))
						
						dbdl = sqle("SELECT * FROM data WHERE name = '%s' AND addr = '%s';" % (dnsq, dnsa), outp=True)
						
						if (len(dbdl) < 1):
							sqle("INSERT INTO data VALUES ('%s', '%s', '0');" % (dnsq, dnsa))
						
						sqle("UPDATE data SET time = '%s' WHERE name = '%s' AND addr = '%s';" % (epch, dnsq, dnsa))
					
					filt()
					dbdl = cche(dnsq)
					
					if (len(dbdl) > 0):
						debg(addr[0], resl[0][0], dbdl[0][1])
						#a=open("a","w");a.write(dnsr);a.close();
						dnsr = shim(dnsr, dbdl)
						#b=open("b","w");b.write(dnsr);b.close();
				
				sobj.sendto(dnsr, addr)
				break

def priv():
	# get username & os.setgid() & os.setuid()
	pass

if (__name__ == "__main__"):
	sqle("CREATE TABLE data (name VARCHAR(256), addr VARCHAR(256), time VARCHAR(256));")
	priv()
	serv()

Attempting To Finally Contribute

So I haven’t had much to talk about lately due the amount of work given for the last few weeks remaining in school and the building of Fedora 15 for ARMv5 and ARMv7. However, in my spare time, I’ve been trying to contribute positively to the Koji Build System project by submitting mailing list items containing source code unified diff patches. Most of my posts/changes contain feature enhancements requested by other people that I found to be somewhat interesting to code. If you’re interested in viewing the mailing list items, the links can be found below:

Hosts-Page Colour Highlight: [ Ticket ] [ Patch ]
Hosts-Page “Life-Line” Links: [ Ticket ] [ Patch ]
Host-Info Latest-Builds Links: [ Ticket ] [ Patch ]
Koji-Client watch-logs “tail”: [ Ticket ] [ Patch ]
Koji-Hub RPMTAG_SIGSHA2 Hashing: [ Ticket ] [ Patch ]

PHP Koji Stats Page

Note: put this in a cron job as it may take a while to run. Use the output to generate a basic html page instead.

stats.php

<?php
	date_default_timezone_set("EST");
	$dta = date(DATE_RFC822);
	$dis = "dist-f15";
	$hua = "http://arm.koji.fedoraproject.org/kojihub";
	$hub = "http://koji.fedoraproject.org/kojihub";
	
	function p($n)
	{
		preg_match("/^(.*)-([^-]+)-([^-]+)$/", $n, $m);
		return $m;
	}
	
	$a = array();
	exec("/usr/bin/koji -s '".$hua."' list-tagged '".$dis."' --latest --quiet | /bin/awk '{ print $1 }'", $a);
	$b = array();
	exec("/usr/bin/koji -s '".$hub."' list-tagged '".$dis."' --latest --quiet | /bin/awk '{ print $1 }'", $b);
	
	$err = array(); $unb = array(); $dif = array(); $wor = array(); $fin = array();
	
	$t = array();
	foreach ($a as $i)
	{
		$n = p($i);
		array_push($t, $n[1]);
	}
	
	$u = array();
	foreach ($b as $i)
	{
		$n = p($i);
		$c = array_search($n[1], $t);
		if ($c === FALSE)
		{
			array_push($u, $i);
		}
		else if ($a[$c] != $i)
		{
			array_push($dif, array($a[$c], $i));
		}
		else if ($a[$c] == $i)
		{
			array_push($fin, $i);
		}
	}
	
	$c = 0; $s = ""; $l = count($u); $v = array();
	for ($x = 0; $x < $l; $x += 1)
	{
		$s = ($s."'".$u[$x]."' ");
		$c = ($c + 1);
		if (($c >= 256) or ($x == ($l - 1)))
		{
			exec("/usr/bin/koji -s '".$hua."' buildinfo ".$s." | /bin/grep -Ei '^(genericerror|no such build|build|state): .*$'", $v);
			$c = 0; $s = "";
		}
	}
	array_push($v, "");
	
	$b = ""; $s = "";
	foreach ($v as $i)
	{
		$m = array(); $n = array();
		if (preg_match("/^state: (.*)$/i", $i, $m))
		{
			$s = trim($m[1]);
		}
		else if (preg_match("/^.*: ([^ ]*).*$/i", $i, $n) or ($b != ""))
		{
			if ($b != "")
			{
				if ($s == "COMPLETE")
				{
					array_push($fin, $b);
				}
				else if ($s == "BUILDING")
				{
					array_push($wor, $b);
				}
				else if ($s == "")
				{
					array_push($unb, $b);
				}
				else
				{
					array_push($err, $b);
				}
			}
			$b = trim($n[1]); $s = "";
		}
	}
	
	$dtb = date(DATE_RFC822);
?>

<html>
	<head>
		<title>Koji Stats Page</title>
		<style>
			a
			{
				color: black;
				text-decoration: none;
			}
			table
			{
				width: 95%;
			}
			.errb
			{
				background-color: #FF6A6A;
			}
			.unbb
			{
				background-color: #FFA812;
			}
			.difb
			{
				background-color: #FFD700;
			}
			.work
			{
				background-color: #33FF33;
			}
			.buib
			{
				background-color: #499DF5;
			}
			.head
			{
				text-align: left;
				width: 95%;
				display: none;
			}
			.butt
			{
				border: 1px solid black;
				padding-left: 10px;
				padding-right: 10px;
			}
			.list
			{
				border: 1px dotted black;
				text-align: left;
				padding-top: 10px;
				padding-left: 10px;
				width: 95%;
				height: 512px;
				overflow: auto;
			}
		</style>
		<script>
			function show(name)
			{
				var x, objc;
				var elem = ["erro", "unbu", "diff", "work", "buil"];
				for (x = 0; x < elem.length; ++x)
				{
					objc = document.getElementById(elem[x]);
					objc.style.display = "none";
				}
				for (x = 0; x < elem.length; ++x)
				{
					if (name == elem[x])
					{
						objc = document.getElementById(elem[x]);
						objc.style.display = "block";
					}
				}
			}
			function init()
			{
				var objc = document.getElementById("head");
				objc.style.display = "block";
			}
		</script>
	</head>
	<body onload="init();">
		<b><?php print($dis." / ".$dta." / ".$dtb); ?></b>
		<br /><br />
		<center>
			<div id="head" class="head">
				<span class="butt errb"><a href="javascript:show('erro');">Errors [<?php echo count($err); ?>]</a></span>
				<span class="butt unbb"><a href="javascript:show('unbu');">Unbuilt [<?php echo count($unb); ?>]</a></span>
				<span class="butt difb"><a href="javascript:show('diff');">Different [<?php echo count($dif); ?>]</a></span>
				<span class="butt work"><a href="javascript:show('work');">Working [<?php echo count($wor); ?>]</a></span>
				<span class="butt buib"><a href="javascript:show('buil');">Built [<?php echo count($fin); ?>]</a></span>
			</div>
			<br />
			<div id="erro" class="list"><center><table>
				<?php foreach ($err as $i) { print("<tr><td class='butt errb'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
			</table></center></div>
			<div id="unbu" class="list" style="display: none"><center><table>
				<?php foreach ($unb as $i) { print("<tr><td class='butt unbb'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
			</table></center></div>
			<div id="diff" class="list" style="display: none"><center><table>
				<?php foreach ($dif as $i) { print("<tr><td class='butt difb'><a href=\"javascript:logs('".$i[0]."');\">".$i[0]." [".$i[1]."]</a></span></td></tr>\n"); } ?>
			</table></center></div>
			<div id="work" class="list" style="display: none"><center><table>
				<?php foreach ($wor as $i) { print("<tr><td class='butt work'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
			</table></center></div>
			<div id="buil" class="list" style="display: none"><center><table>
				<?php foreach ($fin as $i) { print("<tr><td class='butt buib'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
			</table></center></div>
		</center>
	</body>
</html>

2x Random Scripts

Koji Build Que’r

qu.sh
#!/bin/bash
export l=`find ~/Desktop/f15v5.0 -type f | grep -i '\.src\.rpm' | wc -l`
export x=1
while true
do
	ak list-tagged dist-f15 > /tmp/t
	while true
	do
		ak list-tasks --mine --quiet | grep '^[0-9]' | grep -Ei ' (open|free) .* build' > /tmp/n
		#echo "got tasks..." ; cat /tmp/n | wc -l ; echo
		if [ `cat /tmp/n | wc -l` -ge 10 ]
		then
			break
		fi
		p=`find ~/Desktop/f15v5.0 -type f | grep -i '\.src\.rpm' | head -n "$x" | tail -n 1`
		q=`basename "$p" | sed -e 's/[^0-9A-Za-z]/./g' -e 's/\.src\.rpm//g'`
		#echo "checking pkg [$p] [$q]..." ; echo
		c=`cat /tmp/n /tmp/t | grep -i "$q"`
		let x="($x % $l) + 1"
		if [ "$c" != "" ]
		then
			continue
		fi
		echo "queing [$p] skipped [$x]"
		ak build dist-f15 "$p" --nowait
		let n="$n + 1"
	done
	sleep 60
done

and a javascript host highlighter

hosts.js
// ==UserScript==
// @name           hosts
// @namespace      builds
// @include        http://arm.koji.fedoraproject.org/koji/hosts?order=last_update&state=enabled
// ==/UserScript==

var dateob = new Date();
var yerstr = (dateob.getFullYear() + "");
var monstr = ((dateob.getMonth() + 1) + "");
var daystr = (dateob.getDate() + "");
var srcobj = document.getElementById("loginInfo");
var scdate = (parseInt(yerstr + monstr + daystr + srcobj.innerHTML.replace(/[^0-9]/g, "").substring(6)) / 100);
var dsttab = document.getElementsByClassName("data-list");
var dstrow = dsttab[0].getElementsByTagName("tr");

for (var x = 0; x < dstrow.length; ++x)
{
	var dstcol = dstrow[x].getElementsByTagName("td");
	
	if (dstcol.length > 5)
	{
		var chdate = (parseInt(dstcol[5].innerHTML.replace(/[^0-9]/g, "")) / 100);
		
		if ((scdate - 5) >= chdate)
		{
			var lnklst = dstcol[1].getElementsByTagName("a");
			
			for (var y = 0; y < lnklst.length; ++y)
			{
				lnklst[y].innerHTML = ("<font color='red'>" + lnklst[y].innerHTML + "</font>");
			}
		}
	}
}

Repo Query In Shell Script (It’s Slow Tho)

$ ./query.sh "http://australia.proximity.on.ca/fedora-arm/f15v5.0/repo" "pth"
pth-2.0.7-10.src.rpm
pth-2.0.7-13.fc15.src.rpm
*:pth-2.0.7-13.fc15.src.rpm
#!/bin/bash
prim=`curl -s "${1}/repodata/repomd.xml" | tr -d '\t\r\n' | gawk '{ gsub(/<data type=/, "\n<data type="); print; }' | grep -i '<data type="primary">' | sed -e 's@.*<location href="\([^"]*\)"/>.*@\1@g'`
curl -s "${1}/${prim}" > /tmp/repo.data.gz
rm -f /tmp/repo.data 2> /dev/null
gunzip /tmp/repo.data.gz
if [ "${3}" == "f" ]
then
	rm -f /tmp/repo.line 2> /dev/null
fi
if [ ! -f /tmp/repo.line ]
then
	cat /tmp/repo.data | tr -d '\t\r\n' | gawk '{ gsub(/<package type=/, "\n<data type="); print; }' > /tmp/repo.line
fi
safe=`echo "${2}" | sed -e 's/[^0-9A-Za-z]/./g'`
pkgs=`cat /tmp/repo.line | grep -i "<name>${safe}</name>" | sed -e 's@.*<rpm:sourcerpm>\([^<]*\)</rpm:sourcerpm>.*@\1@g'`
if [ "${pkgs}" == "" ]
then
	echo "x:${2}"
	exit 1
fi
echo "${pkgs}"
x=2
leng=`echo "${pkgs}" | wc -l`
high=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $1 }'`
while [ $x -le $leng ]
do
	comp=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $'$x' }'`
	rpmdev-vercmp "$high" "$comp" > /dev/null 2>&1
	if [ $? -eq 12 ]
	then
		high="$comp"
	fi
	let x="$x + 1"
done
echo "*:${high}"

A “Bootstrapable” Koji : A “Feature” Bypassed

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm
282 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm
290 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm 
298 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko list-tagged builds nled
Build                                     Tag                   Built by
----------------------------------------  --------------------  ----------------
nled-2.52-7.267919.111107.fc15            builds                admin
nled-2.52-7.496d81.111107.fc15            builds                admin
nled-2.52-7.fc15                          builds                admin

# ls -l /mnt/koji/packages/nled/2.52
total 12
drwxr-xr-x 6 apache apache 4096 Nov  7 19:43 7.267919.111107.fc15
drwxr-xr-x 6 apache apache 4096 Nov  7 19:52 7.496d81.111107.fc15
drwxr-xr-x 6 apache apache 4096 Nov  7 20:01 7.fc15

...

# ko list-tagged builds --latest nled
Build                                     Tag                   Built by
----------------------------------------  --------------------  ----------------
nled-2.52-7.fc15                          builds                admin

# yum clean all ; yum --disablerepo=* --enablerepo=custom-test list nled
nled.x86_64  2.52-7.fc15  custom-test

Code change: ( Don’t forget: import hashlib )

> ./hub/kojihub.py

def new_build(data):
    """insert a new build entry"""
    tdic = data.copy()
    data = data.copy()
    if not data.has_key('pkg_id'):
        #see if there's a package name
        name = data.get('name')
        if not name:
            raise koji.GenericError, "No name or package id provided for build"
        data['pkg_id'] = new_package(name,strict=False)
    for f in ('version','release','epoch'):
        if not data.has_key(f):
            raise koji.GenericError, "No %s value for build" % f
    #provide a few default values
    data.setdefault('state',koji.BUILD_STATES['COMPLETE'])
    data.setdefault('completion_time', 'NOW')
    data.setdefault('owner',context.session.user_id)
    data.setdefault('task_id',None)
    #check for existing build
    # TODO - table lock?
    q="""SELECT id,state,task_id FROM build
    WHERE pkg_id=%(pkg_id)d AND version=%(version)s AND release=%(release)s
    FOR UPDATE"""
    row = _fetchSingle(q, data)
    if row:
        id, state, task_id = row
        data['id'] = id
        st_desc = koji.BUILD_STATES[state]
        if st_desc == 'BUILDING':
            koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            # check to see if this is the controlling task
            if data['state'] == state and data.get('task_id','') == task_id:
                #the controlling task must have restarted (and called initBuild again)
                return id
            raise koji.GenericError, "Build already in progress (task %d)" % task_id
            # TODO? - reclaim 'stale' builds (state=BUILDING and task_id inactive)
        elif st_desc in ('FAILED','CANCELED'):
            koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            #should be ok to replace
            update = """UPDATE build SET state=%(state)i,task_id=%(task_id)s,
            owner=%(owner)s,completion_time=%(completion_time)s,create_event=get_event()
            WHERE id = %(id)i"""
            _dml(update, data)
            koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            return id
        else:
            # BEG METHOD HIJACK
            # open the current file
            frot = "/mnt/koji/packages"
            fpth = ("%s/%s/%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
            fnme = ("%s-%s-%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
            fsrc = ("%s/%s/src/%s.src.rpm" % (frot, fpth, fnme))
            try:
                fobj = open(fsrc, "r")
            except:
                raise koji.GenericError, "Could not open file [%s]" % (fsrc)
            # hash the current file
            hobj = hashlib.md5()
            while (1):
                fdta = fobj.read(2**10)
                if (not fdta):
                    break
                hobj.update(fdta)
            fobj.close()
            hash = hobj.hexdigest()
            tida = time.strftime("%y%m%d")
            # set the new release name
            while (1):
                rell = str(data["release"]).rsplit(".", 1)
                rell.insert(1, hash[0:6] + "." + tida)
                reln = ".".join(rell)
                foln = ("%s/%s/%s/%s" % (frot, str(data["name"]), str(data["version"]), reln))
                if (not os.path.exists(foln)):
                    break
                hobj = hashlib.md5()
                hobj.update(hash)
                hash = hobj.hexdigest()
            data["reln"] = reln
            # get a list of old files/folders
            fill = []
            dirl = ["%s/%s" % (frot, fpth)]
            while (len(dirl) > 0):
                try:
                    tmpl = os.listdir(dirl[0])
                except:
                    tmpl = []
                parn = dirl[0]
                dirl.pop(0)
                for tmpi in tmpl:
                    tpth = ("%s/%s" % (parn, tmpi))
                    if (os.path.isdir(tpth)):
                        dirl.append(tpth)
                    else:
                        name = re.sub("[^0-9A-Za-z]", ".", str(data["name"]))
                        vers = re.sub("[^0-9A-Za-z]", ".", str(data["version"]))
                        rels = re.sub("[^0-9A-Za-z]", ".", str(data["release"]))
                        robj = re.match("^(.*)%s(.*)%s(.*)%s(.*)$" % (name, vers, rels), tmpi)
                        if (robj):
                            newn = (parn + "/" + robj.group(1) + name + robj.group(2) + vers + robj.group(3) + reln + robj.group(4))
                            fill.append([tpth, newn])
            fill.append([frot + "/" + fpth, foln])
            # update the sql database
            c = context.cnx.cursor()
            buil = "UPDATE build SET release=%(reln)s WHERE id = %(id)i"
            #_dml(buil, data)
            c.execute(buil, data)
            rpmi = "UPDATE rpminfo SET release=%(reln)s WHERE build_id = %(id)i"
            #_dml(rpmi, data)
            c.execute(rpmi, data)
            context.cnx.commit()
            c.close()
            # rename the files/folders now
            for item in fill:
                os.rename(item[0], item[1])
            # carry on the same now
            return new_build(tdic)
            # END METHOD HIJACK
    else:
        koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
    #insert the new data
    data['id'] = _singleValue("SELECT nextval('build_id_seq')")
    q="""
    INSERT INTO build (id,pkg_id,version,release,epoch,state,
            task_id,owner,completion_time)
    VALUES (%(id)i,%(pkg_id)i,%(version)s,%(release)s,%(epoch)s,
            %(state)s,%(task_id)s,%(owner)s,%(completion_time)s)
    """
    _dml(q, data)
    koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
    #return build_id
    return data['id']

createrepo From Scratch Using Forks & Pipes

note: this is experimental/proof-of-concept code for timing purposes

This script tries to implement very basic createrepo functionality and is based on a model of forks and pipes for distributed, inter-process communication.

repo.py

import hashlib
import multiprocessing
import os
import re
import select
import sys

def rpmhinfo(p, n):
	# file
	f = open(n, "r")
	d = f.read()
	f.close()
	o = []
	# lead
	magic = d[0:4]
	vers = d[4:6]
	rtype = ((ord(d[6]) << 8) + ord(d[7]))
	rarch = d[8:10]
	name = d[10:76].strip("\0\t\r\n ")
	osnum = d[76:78]
	sigt = d[78:80]
	resr = d[80:96]
	o.append([magic, vers, rtype, rarch, name, osnum, sigt, resr])
	if (rtype != 0):
		return None
	# extra
	q = n[len(p):]; o[0].append(q)
	s = len(d); o[0].append(s)
	h = hashlib.sha256(d).hexdigest(); o[0].append(h)
	# loop
	i = 1; m = 96
	while (d[m:m+3] == "\x8e\xad\xe8"):
		# header
		magic = d[m:m+3]
		vers = d[m+3]
		resr = d[m+4:m+8]
		numb = ((ord(d[m+8]) << 24) + (ord(d[m+9]) << 16) + (ord(d[m+10]) << 8) + ord(d[m+11]))
		size = ((ord(d[m+12]) << 24) + (ord(d[m+13]) << 16) + (ord(d[m+14]) << 8) + ord(d[m+15]))
		offs = ((m + 16) + (numb * 16))
		o.append([magic, vers, resr, numb, size, offs, m])
		# index
		x = 0; l = (numb * 16)
		while (x < l):
			tag = ((ord(d[x+m+16]) << 24) + (ord(d[x+m+17]) << 16) + (ord(d[x+m+18]) << 8) + ord(d[x+m+19]))
			rtype = ((ord(d[x+m+20]) << 24) + (ord(d[x+m+21]) << 16) + (ord(d[x+m+22]) << 8) + ord(d[x+m+23]))
			offs = ((ord(d[x+m+24]) << 24) + (ord(d[x+m+25]) << 16) + (ord(d[x+m+26]) << 8) + ord(d[x+m+27]))
			numb = ((ord(d[x+m+28]) << 24) + (ord(d[x+m+29]) << 16) + (ord(d[x+m+30]) << 8) + ord(d[x+m+31]))
			data = ""
			y = (o[i][5] + offs); j = 0
			while (j < numb):
				if (rtype == 3):
					data += (str((ord(d[y+0]) << 8) + ord(d[y+1])) + "\n")
					y += 2; j += 1
				elif (rtype == 4):
					data += (str((ord(d[y+0]) << 24) + (ord(d[y+1]) << 16) + (ord(d[y+2]) << 8) + ord(d[y+3])) + "\n")
					y += 4; j += 1
				elif ((rtype == 6) or (rtype == 8) or (rtype == 9)):
					while (d[y] != "\0"):
						data += d[y]
						y += 1
					data += "\n"
					y += 1; j += 1
				elif (rtype == 7):
					data += d[y]
					y += 1; j += 1
			x += 16
			o[i].append([tag, rtype, offs, numb, data])
		m = (o[i][4] + o[i][5])
		while ((m % 8) != 0):
			m += 1
		i += 1
	# return
	return o

def safestri(inptstri):
	outpstri = inptstri
	outpstri = outpstri.replace("&", "&amp;")
	outpstri = outpstri.replace("<", "&lt;")
	outpstri = outpstri.replace(">", "&gt;")
	outpstri = outpstri.strip()
	
	return outpstri

def makedict(rpmhobjc):
	i = (len(rpmhobjc) - 1)
	infodict = {"locn":rpmhobjc[0][8], "size":rpmhobjc[0][9], "hash":rpmhobjc[0][10], "poch":0, "pkgr":"", "urln":"", "vndr":"", "heds":rpmhobjc[i][6], "hede":(rpmhobjc[i][4]+rpmhobjc[i][5])}
	
	for x in range(1, len(rpmhobjc)):
		for y in range(7, len(rpmhobjc[x])):
			#print(rpmhobjc[x][y])
			if ((rpmhobjc[x][y][0] == 1000) and (rpmhobjc[x][y][1] == 6)):
				infodict["name"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1022) and (rpmhobjc[x][y][1] == 6)):
				infodict["arch"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1003) and (rpmhobjc[x][y][1] == 4)):
				infodict["poch"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1001) and (rpmhobjc[x][y][1] == 6)):
				infodict["vers"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1002) and (rpmhobjc[x][y][1] == 6)):
				infodict["rels"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1004) and (rpmhobjc[x][y][1] == 9)):
				infodict["summ"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1005) and (rpmhobjc[x][y][1] == 9)):
				infodict["desc"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1015) and (rpmhobjc[x][y][1] == 6)):
				infodict["pkgr"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1020) and (rpmhobjc[x][y][1] == 6)):
				infodict["urln"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1006) and (rpmhobjc[x][y][1] == 4)):
				infodict["RPMTAG_BUILDTIME"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1009) and (rpmhobjc[x][y][1] == 4)):
				infodict["RPMTAG_SIZE"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 4)):
				infodict["SIGTAG_PAYLOADSIZE"] = safestri(rpmhobjc[x][y][4])
			
			elif ((rpmhobjc[x][y][0] == 1014) and (rpmhobjc[x][y][1] == 6)):
				infodict["licn"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1011) and (rpmhobjc[x][y][1] == 6)):
				infodict["vndr"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1016) and (rpmhobjc[x][y][1] == 9)):
				infodict["grup"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 6)):
				infodict["host"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1044) and (rpmhobjc[x][y][1] == 6)):
				infodict["srpm"] = safestri(rpmhobjc[x][y][4])
			
			elif ((rpmhobjc[x][y][0] == 1047) and (rpmhobjc[x][y][1] == 8)):
				infodict["prvn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1113) and (rpmhobjc[x][y][1] == 8)):
				infodict["prvl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1112) and (rpmhobjc[x][y][1] == 4)):
				infodict["prvf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1049) and (rpmhobjc[x][y][1] == 8)):
				infodict["reqn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1050) and (rpmhobjc[x][y][1] == 8)):
				infodict["reql"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1048) and (rpmhobjc[x][y][1] == 4)):
				infodict["reqf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1054) and (rpmhobjc[x][y][1] == 8)):
				infodict["conn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1055) and (rpmhobjc[x][y][1] == 8)):
				infodict["conl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1053) and (rpmhobjc[x][y][1] == 4)):
				infodict["conf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1090) and (rpmhobjc[x][y][1] == 8)):
				infodict["obsn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1115) and (rpmhobjc[x][y][1] == 8)):
				infodict["obsl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1114) and (rpmhobjc[x][y][1] == 4)):
				infodict["obsf"] = rpmhobjc[x][y][4].split("\n")
			
			elif ((rpmhobjc[x][y][0] == 1118) and (rpmhobjc[x][y][1] == 8)):
				infodict["flep"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1117) and (rpmhobjc[x][y][1] == 8)):
				infodict["flen"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1116) and (rpmhobjc[x][y][1] == 4)):
				infodict["flei"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1030) and (rpmhobjc[x][y][1] == 3)):
				infodict["flet"] = rpmhobjc[x][y][4].split("\n")
	
	return infodict

def procenvr(envrstri):
	outpstri = ""
	
	if (envrstri == ""):
		return outpstri
	
	verslist = envrstri.rsplit("-", 1)
	pochlist = verslist[0].split(":", 1)
	
	if (len(verslist) > 1):
		outpstri = ((" rel=\"%s\"" % (verslist[1])) + outpstri)
	
	if (len(pochlist) < 2):
		pochlist.insert(0, "0")
	
	outpstri = ((" epoch=\"%s\" ver=\"%s\"" % (pochlist[0], pochlist[1])) + outpstri)
	
	return outpstri

def rpmelist(namelist, verslist, flaglist):
	x = len(namelist); y = len(verslist); z = len(flaglist)
	d = {"2":"LT", "4":"GT", "8":"EQ", "10":"LE", "12":"GE"}
	o = ""
	
	if (x < 1):
		return o
	
	if ((x <= y) and (y <= z)):
		f = []
		for w in range(0, x):
			f.append(["", ""])
			
			n = str(int(flaglist[w]) & 0xf)
			try:
				f[w][0] = (" flags=\"%s\"" % (d[n]))
			except:
				pass
			
			n = ((int(flaglist[w]) & 0x200) + (int(flaglist[w]) & 0x100))
			if (n != 0):
				f[w][1] = (" pre=\"1\"")
		
		l = []
		for w in range(0, x):
			if (re.match("^.*rpmlib.*$", namelist[w], re.I)):
				continue
			
			i = procenvr(verslist[w])
			n = ("name=\"%s\"%s%s%s" % (namelist[w], f[w][0], i, f[w][1]))
			
			if (n in l):
				continue
			
			o += ("\t\t\t<rpm:entry %s/>\n" % (n))
			l.append(n)
	
	return o

def makexmld(rpmhdict):
	o = ""
	
	o += ("<package type=\"rpm\">\n")
	
	o += ("\t<name>%s</name>\n" % (rpmhdict["name"]))
	o += ("\t<arch>%s</arch>\n" % (rpmhdict["arch"]))
	o += ("\t<version epoch=\"%s\" ver=\"%s\" rel=\"%s\"/>\n" % (rpmhdict["poch"], rpmhdict["vers"], rpmhdict["rels"]))
	o += ("\t<checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n" % (rpmhdict["hash"]))
	o += ("\t<summary>%s</summary>\n" % (rpmhdict["summ"]))
	o += ("\t<description>%s</description>\n" % (rpmhdict["desc"]))
	o += ("\t<packager>%s</packager>\n" % (rpmhdict["pkgr"]))
	o += ("\t<url>%s</url>\n" % (rpmhdict["urln"]))
	o += ("\t<time file=\"%s\" build=\"%s\"/>\n" % (int(rpmhdict["RPMTAG_BUILDTIME"]) - 10, rpmhdict["RPMTAG_BUILDTIME"]))
	o += ("\t<size package=\"%s\" installed=\"%s\" archive=\"%s\"/>\n" % (rpmhdict["size"], rpmhdict["RPMTAG_SIZE"], rpmhdict["SIGTAG_PAYLOADSIZE"]))
	o += ("\t<location href=\"%s\"/>\n" % (rpmhdict["locn"]))
	
	o += ("\t<format>\n")
	o += ("\t\t<rpm:license>%s</rpm:license>\n" % (rpmhdict["licn"]))
	o += ("\t\t<rpm:vendor>%s</rpm:vendor>\n" % (rpmhdict["vndr"]))
	o += ("\t\t<rpm:group>%s</rpm:group>\n" % (rpmhdict["grup"]))
	o += ("\t\t<rpm:buildhost>%s</rpm:buildhost>\n" % (rpmhdict["host"]))
	o += ("\t\t<rpm:sourcerpm>%s</rpm:sourcerpm>\n" % (rpmhdict["srpm"]))
	o += ("\t\t<rpm:header-range start=\"%s\" end=\"%s\"/>\n" % (rpmhdict["heds"], rpmhdict["hede"]))
	
	try:
		t = rpmelist(rpmhdict["prvn"], rpmhdict["prvl"], rpmhdict["prvf"])
		o += ("\t\t<rpm:provides>\n%s\t\t</rpm:provides>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["reqn"], rpmhdict["reql"], rpmhdict["reqf"])
		o += ("\t\t<rpm:requires>\n%s\t\t</rpm:requires>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["conn"], rpmhdict["conl"], rpmhdict["conf"])
		o += ("\t\t<rpm:conflicts>\n%s\t\t</rpm:conflicts>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["obsn"], rpmhdict["obsl"], rpmhdict["obsf"])
		o += ("\t\t<rpm:obsoletes>\n%s\t\t</rpm:obsoletes>\n" % (t))
	except:
		pass
	
	try:
		x = len(rpmhdict["flen"])
		y = len(rpmhdict["flei"])
		z = len(rpmhdict["flet"])
	except:
		x = -1; y = -2; z = -3
	
	if ((x == y) and (y == z)):
		l = []
		for u in range(0, z):
			if (rpmhdict["flen"][u] == ""):
				break
			
			i = int(rpmhdict["flei"][u])
			pathname = (rpmhdict["flep"][i].rstrip("/") + "/" + rpmhdict["flen"][u])
			if (not re.match("^(/etc.*|.*bin/.*|/usr/lib/sendmail)$", pathname, re.I)):
				continue
			
			if (pathname in l):
				continue
			
			n = (int(rpmhdict["flet"][u]) & 0x4000)
			if (n != 0):
				o += ("\t\t<file type=\"dir\">%s</file>\n" % (pathname))
			
			else:
				o += ("\t\t<file>%s</file>\n" % (pathname))
			
			l.append(pathname)
	
	o += ("\t</format>\n")
	
	o += ("</package>\n")
	
	return [o]

def findfile(foldname):
	x = 0
	filelist = [foldname]
	remolist = []
	
	while (x < len(filelist)):
		if (os.path.isdir(filelist[x])):
			templist = os.listdir(filelist[x])
			
			for tempitem in templist:
				filelist.append(filelist[x] + "/" + tempitem)
			
			remolist.append(filelist[x])
		
		else:
			if (not re.match("^.*\.rpm$", filelist[x], re.I)):
				remolist.append(filelist[x])
			
			elif (re.match("^(%s/repodata/.*|.*debuginfo.*|.*\.src\.rpm)$" % (foldname), filelist[x], re.I)):
				remolist.append(filelist[x])
		
		x += 1
	
	for remoitem in remolist:
		while (filelist.count(remoitem) > 0):
			filelist.remove(remoitem)
	
	return filelist

def main():
	if (len(sys.argv) < 2):
		print("Usage: %s <repo-fold>" % (sys.argv[0]))
		sys.exit(0)
	
	repofold = sys.argv[1].rstrip("/")
	repolist = findfile(repofold)
	repoleng = len(repolist)
	
	print("found [%d] rpm files..." % (repoleng))
	
	childnum = 3
	destfold = "/tmp/repodata"
	pipelist = []; proclist = []
	
	try:
		os.mkdir(destfold)
	except:
		pass
	
	print("beg primary.xml...")
	
	for x in range(0, childnum):
		(parnpipe, chilpipe) = multiprocessing.Pipe(True)
		procnumb = os.fork()
		
		if (procnumb == 0):
			parnpipe.close()
			fileobjc = open("%s/primary.%d.xml" % (destfold, x), "w")
			
			while (1):
				chilpipe.send("more")
				filename = chilpipe.recv()
				
				if (not filename):
					break
				
				print("[%d] : %s" % (x, filename))
				headobjc = rpmhinfo(repofold + "/", filename)
				
				try:
					headdict = makedict(headobjc)
					xmldobjc = makexmld(headdict)
					fileobjc.write(xmldobjc[0])
				except:
					sys.stderr.write("erro:" + str(sys.exc_info()) + "\n")
					pass
			
			fileobjc.close()
			sys.exit(0)
		
		else:
			chilpipe.close()
			pipelist.append(parnpipe)
			proclist.append(procnumb)
	
	while (len(pipelist) > 0):
		(readlist, outplist, errolist) = select.select(pipelist, [], [])
		
		for pipeitem in pipelist:
			if (pipeitem not in readlist):
				continue
			
			try:
				dumydata = pipeitem.recv()
				
				if (len(repolist) > 0):
					pipeitem.send(repolist[0])
					repolist.pop(0)
				else:
					pipeitem.send("")
			
			except:
				pipelist.remove(pipeitem)
	
	for procitem in proclist:
		try:
			os.waitpid(procitem, 0)
		except:
			pass
	
	print("end primary.xml...")
	
	primoutp = ""
	for x in range(0, childnum):
		try:
			fileobjc = open("%s/primary.%d.xml" % (destfold, x), "r")
			primoutp += fileobjc.read()
			fileobjc.close()
			os.unlink("%s/primary.%d.xml" % (destfold, x))
		except:
			pass
	
	fileobjc = open("%s/primary.xml" % (destfold), "w")
	fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
	fileobjc.write("<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n" % (repoleng))
	fileobjc.write(primoutp)
	fileobjc.write("</metadata>\n")
	fileobjc.close()
	
	fileobjc = open("%s/repomd.xml" % (destfold), "w")
	fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
	fileobjc.write("<repomd xmlns=\"http://linux.duke.edu/metadata/repo\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\">\n")
	fileobjc.write("<data type=\"primary\">\n")
	fileobjc.write("\t<checksum type=\"sha256\">%s</checksum>\n" % (hashlib.sha256(primoutp).hexdigest()))
	fileobjc.write("\t<location href=\"repodata/primary.xml\"/>\n")
	fileobjc.write("\t<size>%d</size>\n" % (len(primoutp)))
	fileobjc.write("</data>\n")
	fileobjc.write("</repomd>\n")
	fileobjc.close()

if (__name__ == "__main__"):
	main()

Hijacking The Mock Yum Buildroot

So I wanted to see if I could speed up common mock tasks without using their built-in caching methods as the buildroot package set is changing and it may not pick up those changes. The mock source code is located here (https://fedorahosted.org/mock/) and the results below show that on an x64 machine, the nled build went 22.5% faster! The downside is that you would have to set aside a dedicate builder to constantly create and tar the updated buildroots for the builders running the modified mock. Also note that the speed-up shown for the modified mock did not have to download the tar as it was locally stored.

Regular mock

# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time mock -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm 
INFO: mock.py version 1.1.9 starting...
...
INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 1 minutes 36 seconds
...
DEBUG: kill orphans

real	1m37.880s
user	1m3.519s
sys	0m4.444s

Modified mock

# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time ./py/mock.py -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm 
INFO: mock.py version unreleased_version starting...
...
INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 0 minutes 21 seconds
...
DEBUG: kill orphans

real	0m22.001s
user	0m44.526s
sys	0m2.925s

./py/mockbuild/backend.py

155:    decorate(traceLog())
156:    def _unlock_and_rm_chroot(self):
157:        if (os.path.exists("/tmp/buildroot.flag")):
158:            print("CUSTOM: HIJACK SAVING [%s]" % (self.makeChrootPath()))
159:            os.system("tar -cf \"/tmp/buildroot.tar\" -C '%s' ." % (self.makeChrootPath()))
160:            try:
161:                os.unlink("/tmp/buildroot.flag")
162:            except:
163:                pass
...
826:    decorate(traceLog())
827:    def _yum(self, cmd, returnOutput=0):
828:        """use yum to install packages/package groups into the chroot"""
829:        flag=0
830:        try:
831:            if ((cmd[0] == "groupinstall") and (cmd[1] == "buildsys-build")):
832:                flag=1
833:                if (os.path.exists("/tmp/buildroot.tar")):
834:                    os.system("tar -xvf /tmp/buildroot.tar -C '%s'" % (self.makeChrootPath()))
835:                    return ""
836:        except:
837:            pass
...
853:        try:
854:            self._callHooks("preyum")
855:            output = mockbuild.util.do(yumcmd, returnOutput=returnOutput)
856:            self._callHooks("postyum")
857:            if (flag == 1):
858:                os.system("touch /tmp/buildroot.flag")
859:                raise mockbuild.exception.YumError, "CUSTOM: HIJACK YUM ERROR"
860:            else:
861:                return output
862:        except mockbuild.exception.Error, e:
863:            raise mockbuild.exception.YumError, str(e)

Repo Management Helper Script

So we’re finally on to building Fedora 15 for ARMv5 on F13 builders and since we’re managing a repo manually, I needed a script to do the following:

  • Move all failed build packages back to the source folder to be rebuilt again
  • Move all completed build packages not in the destination to that directory
  • Move any build packages in the destination that are not built back to the source folder

Here is the following script which helps us achieve this:

fixs.py

import os
import re
import subprocess

z = '/var/export/f15v5.0'
os.system("mkdir \"%s/fixs/\" 2> /dev/null" % (z))

r = subprocess.Popen(["find", "%s/repo" % (z), "-type", "f"], stdout=subprocess.PIPE).communicate()[0]
s = r.split("\n")

f = subprocess.Popen(["ls", "%s/dest" % (z)], stdout=subprocess.PIPE).communicate()[0]
g = f.split("\n")

for i in g:
	i = i.strip()
	n = re.sub("\.src\.rpm$", "", i)
	m = [0, 0]
	for j in s:
		j = j.strip()
		k = re.sub("[^0-9A-Za-z]", ".", n)
		if (re.match("^.*%s.*$" % (k), j)):
			m[0] = 1
		if (re.match("^.*%s\.src\.rpm$" % (k), j)):
			m[1] = 1
	c = ("mv \"%s/dest/%s\" \"%s/fixs/\"" % (z, i, z))
	if ((m[0] == 1) and (m[1] == 0)):
		print(c)
		os.system(c)
	if ((m[0] == 0) and (m[1] == 0)):
		print(c)
		os.system(c)

Mini-Koji Build System (Moji)

This was a small project used to help with the initial boot-strapping and rebuild of the various Fedora-ARM packages without reliance on previous Fedora builds. We were unable to do this with Koji because it only lets you build a single version of a package once. This was a problem because we used F13 builders to build F15 and then we upgrade the builders to F15 so that they could rebuild F15 once more. This was done to help produce the cleanest and most native possible build that we could for F15 for both ARMv5 and ARMv7.

Edit: I also did a little Koji mod which could bypass the “one-build” restriction here

Source Code (Git Hub)
Binary Program (Fedora People)

Fedora ARM On A Panda From Scratch

Below is a script to help create a panda sdcard from scratch but using pre-compiled kernel and boot files.

#!/bin/bash

if [ "$1" == "" ]
then
	echo "Usage: $0 <device>"
	exit 1
fi

# http://www.angstrom-distribution.org/demo/pandaboard/mkcard.txt

export CARD="$1"
export BOOT="/mnt/boot"
export ROOT="/mnt/root"
export TEMP="/tmp/temp"

echo "formatting [${CARD}] after sleeping..." ; sleep 10
mkdir ${TEMP} 2> /dev/null
umount ${CARD}{1..4} 2> /dev/null
dd if=/dev/zero of=${CARD} bs=1024 count=1024
./mkcard.sh ${CARD}

# http://www.angstrom-distribution.org/demo/pandaboard/{MLO,u-boot.bin,uImage}

umount ${BOOT} 2> /dev/null ; mkdir ${BOOT} 2> /dev/null ; mount ${CARD}1 ${BOOT}

wget -O ${BOOT}/MLO 'http://www.angstrom-distribution.org/demo/pandaboard/MLO' ; sync
wget -O ${BOOT}/u-boot.bin 'http://www.angstrom-distribution.org/demo/pandaboard/u-boot.bin'
wget -O ${BOOT}/uImage 'http://www.angstrom-distribution.org/demo/pandaboard/uImage'

time sync ; umount ${BOOT} 2> /dev/null

# http://fedoraproject.org/wiki/Architectures/ARM
# http://scotland.proximity.on.ca/fedora-arm/rootfs/f13-rc1/rootfs-f13-rc1-2011-06-29.tar.bz2

umount ${ROOT} 2> /dev/null ; mkdir ${ROOT} 2> /dev/null ; mount ${CARD}2 ${ROOT}

tar -xjvf rootfs-f*.tar.bz2 -C ${ROOT} ; mv ${ROOT}/rootfs-*/* ${ROOT} 2> /dev/null

time sync ; umount ${ROOT} 2> /dev/null

# path to kernel modules

umount ${ROOT} 2> /dev/null ; mkdir ${ROOT} 2> /dev/null ; mount ${CARD}2 ${ROOT}

echo '# custom added section' >> ${ROOT}/etc/rc.local
echo '/sbin/ifconfig usb0 inet 192.168.1.253 netmask 255.255.255.0 up' >> ${ROOT}/etc/rc.local
echo '/sbin/route add default gw 192.168.1.254' >> ${ROOT}/etc/rc.local
echo 'while true ; do /sbin/agetty -L 115200 console vt100 ; done' >> ${ROOT}/etc/rc.local

time sync ; umount ${ROOT} 2> /dev/null

# done

eject ${CARD}

Note: A good guide to compiling the kernel is located here: http://wiki.meego.com/ARM/OMAP4_Panda

Basic Stats Page Outside Of Koji

So since we’re building Fedora ARM packages outside of Koji first (on F13 builders and then again on F15 builders), I decided to create a simple stats page so our team could track its status.

<?php
	function fileread($filename)
	{
		$outpstri = "";
		$fileobjc = fopen($filename, "r");
		$outpstri .= fread($fileobjc, filesize($filename));
		fclose($fileobjc);
		$outpstri .= "\n";
		return $outpstri;
	}
	$foldname = "./done";
	if (isset($_GET["pkgn"]))
	{
		$outpstri = "";
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/build.log");
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/root.log");
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/state.log");
		$outpstri = preg_replace("/&/", "&amp;", $outpstri);
		$outpstri = preg_replace("/</", "&lt;", $outpstri);
		$outpstri = preg_replace("/>/", "&gt;", $outpstri);
		$outpstri = str_replace("\n", " <br /> \n ", $outpstri);
		print($outpstri."\n");
		die;
	}
	exec("find ".$foldname." -type f", $outplist);
	$donestri = ""; $donelist = array();
	$errostri = ""; $errolist = array();
	$totllist = array();
	foreach ($outplist as $outpitem)
	{
		$safefold = preg_replace("/([\/.])/", "\\\\$1", $foldname);
		if (!preg_match("/^".$safefold."\/mock\.([^\/]+)\/.*$/", $outpitem))
		{
			continue;
		}
		$pkgsname = preg_replace("/^".$safefold."\/mock\.([^\/]+)\/.*$/", "$1", $outpitem);
		if (preg_match("/^.*\.src\.rpm$/", $outpitem))
		{
			if (!in_array($pkgsname, $donelist))
			{
				array_push($donelist, $pkgsname);
			}
		}
		if (!in_array($pkgsname, $totllist))
		{
			array_push($totllist, $pkgsname);
		}
	}
	foreach ($totllist as $totlitem)
	{
		if (!in_array($totlitem, $donelist))
		{
			array_push($errolist, $totlitem);
		}
	}
	sort($donelist);
	sort($errolist);
	$donenumb = count($donelist);
	$erronumb = count($errolist);
	$donestri .= ("<script>parent.document.title += ' [ ".$donenumb.".d ] ';</script>");
	$donestri .= ("<b>Built: [ ".$donenumb." ]</b> <br /> <br /> \n");
	foreach ($donelist as $doneitem)
	{
		$donestri .= ($doneitem." <br /> \n ");
	}
	$errostri .= ("<script>parent.document.title += ' [ ".$erronumb.".e ] ';</script>");
	$errostri .= ("<b>Error: [ ".$erronumb." ]</b> <br /> <br /> \n");
	foreach ($errolist as $erroitem)
	{
		$errostri .= ("<a href=\"javascript:function a() { parent.document.getElementById('logswind').src = ('".$_SERVER["PHP_SELF"]."?pkgn=".$erroitem."'); } a();\">".$erroitem."</a> <br /> \n ");
	}
	if (isset($_GET["done"]))
	{
		print($donestri);
		die;
	}
	if (isset($_GET["erro"]))
	{
		print($errostri);
		die;
	}
?>
<html>
	<head>
		<title>loading...</title>
		<style>
			table
			{
				width: 95%;
			}
			iframe
			{
				border: 1px dotted black;
				width: 100%;
			}
			#errowind
			{
				height: 160px;
			}
			#logswind
			{
				height: 240px;
			}
			#donewind
			{
				height: 160px;
			}
		</style>
		<script>
			function pkgslist()
			{
				document.title = "";
				document.getElementById("errowind").src = ('<?php echo $_SERVER["PHP_SELF"]; ?>?erro=true');
				document.getElementById("donewind").src = ('<?php echo $_SERVER["PHP_SELF"]; ?>?done=true');
			}
			function pageinit()
			{
				setInterval("pkgslist();", 30 * 1000);
			}
		</script>
	</head>
	<body onload="pkgslist(); pageinit();">
		<b>Date: [ <?php echo date("y-m-d H:i:s"); ?> ]</b>
		<br /> <br />
		<center>
		<table>
			<tr>
				<td><iframe id="errowind"></iframe></td>
			</tr>
			<tr>
				<td> &nbsp; </td>
			</tr>
			<tr>
				<td><b>Logs:</b></td>
			</tr>
			<tr>
				<td><iframe id="logswind"></iframe></td>
			</tr>
			<tr>
				<td> &nbsp; </td>
			</tr>
			<tr>
				<td><iframe id="donewind"></iframe></td>
			</tr>
		</table>
		</center>
	</body>
</html>

F15 ARMv5 Building On F13 Builders

This script can be distributed to multiple builders to build the various Fedora packages with mock. The setup commands for the server side are included (but commented) in the script below. Note: The script now requires an apache server to be running which allows for the new fedora-release package information to be retrieved for correct build tagging.

> httpd.conf

Alias /builds/ "/tmp/builds/"

<Directory "/tmp/builds">
    Options Indexes MultiViews FollowSymLinks
    AllowOverride None
    Order allow,deny
    Allow from all
</Directory>

> arm-builder.sh

#!/bin/bash

# COMMON: variable definitions for this script

SSHDHOST="1.2.3.4"
SSHDUSER="builder"
SSHDPASS="/tmp/sshkey"
SSHDROOT="/tmp/builds"
LOCLTEMP="/tmp/work"

SSHDPRIO="${SSHDROOT}/prio"
SSHDPKGS="${SSHDROOT}/pkgs"
SSHDWORK="${SSHDROOT}/work"
SSHDDONE="${SSHDROOT}/done"

# SERVER: local and remote account setup

#passwd ; chsh -s /bin/bash
#mkdir -p ~/.ssh ; echo 'ssh-pub-key@local' > ~/.ssh/authorized_keys
#chmod 700 ~/.ssh ~/.ssh/authorized_keys

# SERVER: build directory creation

#mkdir -p ${SSHDPRIO} ${SSHDPKGS} ${SSHDWORK} ${SSHDDONE}

# SERVER: create intial build package files

#koji list-tagged dist-f15 --latest > /tmp/list.txt
#cd ${SSHDPKGS} ; tail -n +3 /tmp/list.txt | awk '{ print $1 }' | while read line ; do touch "$line" ; done

# SERVER: initialize the most basic repo containing F15 release information

#urls="http://kojipkgs.fedoraproject.org/packages/fedora-release/15/1"
#cd ${SSHDDONE} ; wget "${urls}/src/fedora-release-15-1.src.rpm" "${urls}/noarch/fedora-release-15-1.noarch.rpm" "${urls}/noarch/fedora-release-rawhide-15-1.noarch.rpm" ; createrepo --update -d ${SSHDDONE}

# CLIENT: save the ssh key for auto logins

(echo '-----BEGIN ssh-pri-key-----') > ${SSHDPASS}
chmod 700 ${SSHDPASS}

# CLIENT: create a custom fedora repo file for the mock builds

echo "
config_opts['root'] = 'fedora-15-arm'
config_opts['target_arch'] = 'armv5tel'
config_opts['legal_host_arches'] = ('armv5tel', 'armv6l', 'armv7l')
config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
config_opts['dist'] = 'fc15'
config_opts['plugin_conf']['root_cache_enable'] = False
config_opts['plugin_conf']['yum_cache_enable'] = False

config_opts['yum.conf'] = \"\"\"
[main]
cachedir=/var/cache/yum
debuglevel=1
reposdir=/dev/null
logfile=/var/log/yum.log
retries=20
obsoletes=1
gpgcheck=0
assumeyes=1
syslog_ident=mock
syslog_device=

# repos

[custom]
name=fedora-custom
baseurl=http://${SSHDHOST}/builds/done/
enabled=1

### NOTE : USE A VAR HERE FOR THE PATH INSTEAD ###

[f15-arm-koji]
name=fedora15-arm-koji
baseurl=http://arm.koji.fedoraproject.org/repos/dist-f15-build/latest/arm/
enabled=1

[fedora]
name=fedora
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=fedora-13&arch=arm
failovermethod=priority

[updates-released]
name=updates
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f13&arch=arm
failovermethod=priority
\"\"\"
" > /etc/mock/fedora-15-arm.cfg
chmod 755 /etc/mock/fedora-15-arm.cfg

# CLIENT: create a tmp working directory for the pkg builds

mkdir -p ${LOCLTEMP}
chmod 777 ${LOCLTEMP}
cd ${LOCLTEMP}

# CLIENT: ensure that the mock user is in the mock group (not just primary)

useradd -g mock mock
usermod -G mock mock
chsh -s /bin/bash mock

# CLIENT: loop to keep building pkgs

while true
do
	# CLIENT: check for a random pkg and move it out of the directory so it can be worked on
	
	for dirn in ${SSHDPRIO} ${SSHDPKGS}
	do
		pkgn=$(ssh -i ${SSHDPASS} ${SSHDUSER}@${SSHDHOST} "pkgn=\$(ls -t ${dirn} | tail -n 1) ; if [ \"\${pkgn}\" != \"\" ] ; then mv ${dirn}/\${pkgn} ${SSHDWORK}/ ; echo \${pkgn} ; fi")
		
		if [ "${pkgn}" != "" ]
		then
			break
		fi
	done
	
	# CLIENT: remove any old build files
	
	rm -frv ${LOCLTEMP}/*
	
	# CLIENT: if we have a pkg then try to download the source
	
	if [ "${pkgn}" != "" ]
	then
		# CLIENT: download and build the package
		
		echo "Building [${dirn}/${pkgn}] in [`pwd`]"
		
		# CLIENT: download the source rpm pkg from the server
		
		scp -i ${SSHDPASS} ${SSHDUSER}@${SSHDHOST}:${SSHDWORK}/${pkgn} ./
		
		if [ ! -s "${pkgn}" ]
		then
			# CLIENT: pkg prep
			
			rm -frv ${LOCLTEMP}/*
			pkgn=$(echo "${pkgn}" | sed -e 's/\.src\.rpm$//g')
			
			#koji download-build --arch=src "${pkgn}"
			urln=`python -c "import rpmUtils.miscutils ; (n,v,r,e,a)=rpmUtils.miscutils.splitFilename('${pkgn}') ; print('http://kojipkgs.fedoraproject.org/packages/%s/%s/%s.%s/src/${pkgn}.src.rpm' % (n,v,r,a))"`
			wget "${urln}"
		fi
	fi
	
	# CLIENT: check for and make any build files readable
	
	file=$(echo *)
	
	# CLIENT: if we have a source pkg then try to build it
	
	if [ -e "${file}" ]
	then
		# CLIENT: pkg prep
		
		chmod 755 *
		
		# CLIENT: build the source rpm in mock
		
		su -c "mock -vr fedora-15-arm --resultdir=${LOCLTEMP}/mock.${pkgn} ${LOCLTEMP}/${file}" - mock
		
		# CLIENT: copy any mock results back to the server
		
		scp -i ${SSHDPASS} -r mock.${pkgn} ${SSHDUSER}@${SSHDHOST}:${SSHDDONE}/
	fi
	
	sleep 60
done

> repo.py

import os
import sys
import time

repo = "/var/export/f15v5.0/repo"
temp = "/var/export/f15v5.0/temp"

def syse(comd, show=True):
	if (show):
		print(comd)
	
	os.system(comd)

while (1):
	syse("find %s -type f -iname '*.[rx][pm][ml]' -exec ls --time-style='+%%s' -l {} \; 2> /dev/null | tr -s ' ' | cut -d ' ' -f 6- | sort -nr | head -n 1 | grep -i '.*\.rpm$' > /tmp/repo.txt" % (repo))
	
	fobj = open("/tmp/repo.txt", "r")
	chek = fobj.read().strip()
	fobj.close()
	
	if (chek != ""):
		syse("echo \"Starting on [ `date` ]\" | tee -a ~/repo.log")
		
		syse("mkdir \"%s\" 2> /dev/null" % (temp)) # make sure the temp repo directory exists
		syse("rm -fr %s/* 2> /dev/null" % (temp)) # clear the temp repo directory
		
		for item in os.listdir(repo):
			syse("ln -s \"%s/%s\" \"%s/%s\"" % (repo, item, temp, item), show=False)
		
		syse("rm -fr \"%s/repodata\"" % (temp)) # remove any temp repo data links
		syse("cp -fr \"%s/repodata\" \"%s/\"" % (repo, temp)) # copy over the current repo data
		syse("createrepo --update -d \"%s\" --exclude=*debuginfo* --exclude=*.src.rpm" % (temp)) # create/update the new repo data
		syse("rm -fr \"%s/repodata\"" % (repo)) # remove the old repodata files
		syse("cp -fr \"%s/repodata\" \"%s/\"" % (temp, repo)) # copy over the new repo data
		
		syse("echo \"Ending on [ `date` ]\" | tee -a ~/repo.log")
	
	time.sleep(60) # sleep loop or risk fire!