Repo Query In Shell Script (It’s Slow Tho)

$ ./query.sh "http://australia.proximity.on.ca/fedora-arm/f15v5.0/repo" "pth"
pth-2.0.7-10.src.rpm
pth-2.0.7-13.fc15.src.rpm
*:pth-2.0.7-13.fc15.src.rpm
#!/bin/bash
prim=`curl -s "${1}/repodata/repomd.xml" | tr -d '\t\r\n' | gawk '{ gsub(/<data type=/, "\n<data type="); print; }' | grep -i '<data type="primary">' | sed -e 's@.*<location href="\([^"]*\)"/>.*@\1@g'`
curl -s "${1}/${prim}" > /tmp/repo.data.gz
rm -f /tmp/repo.data 2> /dev/null
gunzip /tmp/repo.data.gz
if [ "${3}" == "f" ]
then
	rm -f /tmp/repo.line 2> /dev/null
fi
if [ ! -f /tmp/repo.line ]
then
	cat /tmp/repo.data | tr -d '\t\r\n' | gawk '{ gsub(/<package type=/, "\n<data type="); print; }' > /tmp/repo.line
fi
safe=`echo "${2}" | sed -e 's/[^0-9A-Za-z]/./g'`
pkgs=`cat /tmp/repo.line | grep -i "<name>${safe}</name>" | sed -e 's@.*<rpm:sourcerpm>\([^<]*\)</rpm:sourcerpm>.*@\1@g'`
if [ "${pkgs}" == "" ]
then
	echo "x:${2}"
	exit 1
fi
echo "${pkgs}"
x=2
leng=`echo "${pkgs}" | wc -l`
high=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $1 }'`
while [ $x -le $leng ]
do
	comp=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $'$x' }'`
	rpmdev-vercmp "$high" "$comp" > /dev/null 2>&1
	if [ $? -eq 12 ]
	then
		high="$comp"
	fi
	let x="$x + 1"
done
echo "*:${high}"

Repo Query In Shell Script (It’s Slow Tho)

A “Bootstrapable” Koji : A “Feature” Bypassed

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm
282 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm
290 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko build builds /tmp/nled-2.52-7.fc15.src.rpm 
298 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully

# ko list-tagged builds nled
Build                                     Tag                   Built by
----------------------------------------  --------------------  ----------------
nled-2.52-7.267919.111107.fc15            builds                admin
nled-2.52-7.496d81.111107.fc15            builds                admin
nled-2.52-7.fc15                          builds                admin

# ls -l /mnt/koji/packages/nled/2.52
total 12
drwxr-xr-x 6 apache apache 4096 Nov  7 19:43 7.267919.111107.fc15
drwxr-xr-x 6 apache apache 4096 Nov  7 19:52 7.496d81.111107.fc15
drwxr-xr-x 6 apache apache 4096 Nov  7 20:01 7.fc15

...

# ko list-tagged builds --latest nled
Build                                     Tag                   Built by
----------------------------------------  --------------------  ----------------
nled-2.52-7.fc15                          builds                admin

# yum clean all ; yum --disablerepo=* --enablerepo=custom-test list nled
nled.x86_64  2.52-7.fc15  custom-test

Code change: ( Don’t forget: import hashlib )

> ./hub/kojihub.py

def new_build(data):
    """insert a new build entry"""
    tdic = data.copy()
    data = data.copy()
    if not data.has_key('pkg_id'):
        #see if there's a package name
        name = data.get('name')
        if not name:
            raise koji.GenericError, "No name or package id provided for build"
        data['pkg_id'] = new_package(name,strict=False)
    for f in ('version','release','epoch'):
        if not data.has_key(f):
            raise koji.GenericError, "No %s value for build" % f
    #provide a few default values
    data.setdefault('state',koji.BUILD_STATES['COMPLETE'])
    data.setdefault('completion_time', 'NOW')
    data.setdefault('owner',context.session.user_id)
    data.setdefault('task_id',None)
    #check for existing build
    # TODO - table lock?
    q="""SELECT id,state,task_id FROM build
    WHERE pkg_id=%(pkg_id)d AND version=%(version)s AND release=%(release)s
    FOR UPDATE"""
    row = _fetchSingle(q, data)
    if row:
        id, state, task_id = row
        data['id'] = id
        st_desc = koji.BUILD_STATES[state]
        if st_desc == 'BUILDING':
            koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            # check to see if this is the controlling task
            if data['state'] == state and data.get('task_id','') == task_id:
                #the controlling task must have restarted (and called initBuild again)
                return id
            raise koji.GenericError, "Build already in progress (task %d)" % task_id
            # TODO? - reclaim 'stale' builds (state=BUILDING and task_id inactive)
        elif st_desc in ('FAILED','CANCELED'):
            koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            #should be ok to replace
            update = """UPDATE build SET state=%(state)i,task_id=%(task_id)s,
            owner=%(owner)s,completion_time=%(completion_time)s,create_event=get_event()
            WHERE id = %(id)i"""
            _dml(update, data)
            koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
            return id
        else:
            # BEG METHOD HIJACK
            # open the current file
            frot = "/mnt/koji/packages"
            fpth = ("%s/%s/%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
            fnme = ("%s-%s-%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
            fsrc = ("%s/%s/src/%s.src.rpm" % (frot, fpth, fnme))
            try:
                fobj = open(fsrc, "r")
            except:
                raise koji.GenericError, "Could not open file [%s]" % (fsrc)
            # hash the current file
            hobj = hashlib.md5()
            while (1):
                fdta = fobj.read(2**10)
                if (not fdta):
                    break
                hobj.update(fdta)
            fobj.close()
            hash = hobj.hexdigest()
            tida = time.strftime("%y%m%d")
            # set the new release name
            while (1):
                rell = str(data["release"]).rsplit(".", 1)
                rell.insert(1, hash[0:6] + "." + tida)
                reln = ".".join(rell)
                foln = ("%s/%s/%s/%s" % (frot, str(data["name"]), str(data["version"]), reln))
                if (not os.path.exists(foln)):
                    break
                hobj = hashlib.md5()
                hobj.update(hash)
                hash = hobj.hexdigest()
            data["reln"] = reln
            # get a list of old files/folders
            fill = []
            dirl = ["%s/%s" % (frot, fpth)]
            while (len(dirl) > 0):
                try:
                    tmpl = os.listdir(dirl[0])
                except:
                    tmpl = []
                parn = dirl[0]
                dirl.pop(0)
                for tmpi in tmpl:
                    tpth = ("%s/%s" % (parn, tmpi))
                    if (os.path.isdir(tpth)):
                        dirl.append(tpth)
                    else:
                        name = re.sub("[^0-9A-Za-z]", ".", str(data["name"]))
                        vers = re.sub("[^0-9A-Za-z]", ".", str(data["version"]))
                        rels = re.sub("[^0-9A-Za-z]", ".", str(data["release"]))
                        robj = re.match("^(.*)%s(.*)%s(.*)%s(.*)$" % (name, vers, rels), tmpi)
                        if (robj):
                            newn = (parn + "/" + robj.group(1) + name + robj.group(2) + vers + robj.group(3) + reln + robj.group(4))
                            fill.append([tpth, newn])
            fill.append([frot + "/" + fpth, foln])
            # update the sql database
            c = context.cnx.cursor()
            buil = "UPDATE build SET release=%(reln)s WHERE id = %(id)i"
            #_dml(buil, data)
            c.execute(buil, data)
            rpmi = "UPDATE rpminfo SET release=%(reln)s WHERE build_id = %(id)i"
            #_dml(rpmi, data)
            c.execute(rpmi, data)
            context.cnx.commit()
            c.close()
            # rename the files/folders now
            for item in fill:
                os.rename(item[0], item[1])
            # carry on the same now
            return new_build(tdic)
            # END METHOD HIJACK
    else:
        koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
    #insert the new data
    data['id'] = _singleValue("SELECT nextval('build_id_seq')")
    q="""
    INSERT INTO build (id,pkg_id,version,release,epoch,state,
            task_id,owner,completion_time)
    VALUES (%(id)i,%(pkg_id)i,%(version)s,%(release)s,%(epoch)s,
            %(state)s,%(task_id)s,%(owner)s,%(completion_time)s)
    """
    _dml(q, data)
    koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
    #return build_id
    return data['id']
A “Bootstrapable” Koji : A “Feature” Bypassed

createrepo From Scratch Using Forks & Pipes

note: this is experimental/proof-of-concept code for timing purposes

This script tries to implement very basic createrepo functionality and is based on a model of forks and pipes for distributed, inter-process communication.

repo.py

import hashlib
import multiprocessing
import os
import re
import select
import sys

def rpmhinfo(p, n):
	# file
	f = open(n, "r")
	d = f.read()
	f.close()
	o = []
	# lead
	magic = d[0:4]
	vers = d[4:6]
	rtype = ((ord(d[6]) << 8) + ord(d[7]))
	rarch = d[8:10]
	name = d[10:76].strip("\0\t\r\n ")
	osnum = d[76:78]
	sigt = d[78:80]
	resr = d[80:96]
	o.append([magic, vers, rtype, rarch, name, osnum, sigt, resr])
	if (rtype != 0):
		return None
	# extra
	q = n[len(p):]; o[0].append(q)
	s = len(d); o[0].append(s)
	h = hashlib.sha256(d).hexdigest(); o[0].append(h)
	# loop
	i = 1; m = 96
	while (d[m:m+3] == "\x8e\xad\xe8"):
		# header
		magic = d[m:m+3]
		vers = d[m+3]
		resr = d[m+4:m+8]
		numb = ((ord(d[m+8]) << 24) + (ord(d[m+9]) << 16) + (ord(d[m+10]) << 8) + ord(d[m+11]))
		size = ((ord(d[m+12]) << 24) + (ord(d[m+13]) << 16) + (ord(d[m+14]) << 8) + ord(d[m+15]))
		offs = ((m + 16) + (numb * 16))
		o.append([magic, vers, resr, numb, size, offs, m])
		# index
		x = 0; l = (numb * 16)
		while (x < l):
			tag = ((ord(d[x+m+16]) << 24) + (ord(d[x+m+17]) << 16) + (ord(d[x+m+18]) << 8) + ord(d[x+m+19]))
			rtype = ((ord(d[x+m+20]) << 24) + (ord(d[x+m+21]) << 16) + (ord(d[x+m+22]) << 8) + ord(d[x+m+23]))
			offs = ((ord(d[x+m+24]) << 24) + (ord(d[x+m+25]) << 16) + (ord(d[x+m+26]) << 8) + ord(d[x+m+27]))
			numb = ((ord(d[x+m+28]) << 24) + (ord(d[x+m+29]) << 16) + (ord(d[x+m+30]) << 8) + ord(d[x+m+31]))
			data = ""
			y = (o[i][5] + offs); j = 0
			while (j < numb):
				if (rtype == 3):
					data += (str((ord(d[y+0]) << 8) + ord(d[y+1])) + "\n")
					y += 2; j += 1
				elif (rtype == 4):
					data += (str((ord(d[y+0]) << 24) + (ord(d[y+1]) << 16) + (ord(d[y+2]) << 8) + ord(d[y+3])) + "\n")
					y += 4; j += 1
				elif ((rtype == 6) or (rtype == 8) or (rtype == 9)):
					while (d[y] != "\0"):
						data += d[y]
						y += 1
					data += "\n"
					y += 1; j += 1
				elif (rtype == 7):
					data += d[y]
					y += 1; j += 1
			x += 16
			o[i].append([tag, rtype, offs, numb, data])
		m = (o[i][4] + o[i][5])
		while ((m % 8) != 0):
			m += 1
		i += 1
	# return
	return o

def safestri(inptstri):
	outpstri = inptstri
	outpstri = outpstri.replace("&", "&amp;")
	outpstri = outpstri.replace("<", "&lt;")
	outpstri = outpstri.replace(">", "&gt;")
	outpstri = outpstri.strip()
	
	return outpstri

def makedict(rpmhobjc):
	i = (len(rpmhobjc) - 1)
	infodict = {"locn":rpmhobjc[0][8], "size":rpmhobjc[0][9], "hash":rpmhobjc[0][10], "poch":0, "pkgr":"", "urln":"", "vndr":"", "heds":rpmhobjc[i][6], "hede":(rpmhobjc[i][4]+rpmhobjc[i][5])}
	
	for x in range(1, len(rpmhobjc)):
		for y in range(7, len(rpmhobjc[x])):
			#print(rpmhobjc[x][y])
			if ((rpmhobjc[x][y][0] == 1000) and (rpmhobjc[x][y][1] == 6)):
				infodict["name"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1022) and (rpmhobjc[x][y][1] == 6)):
				infodict["arch"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1003) and (rpmhobjc[x][y][1] == 4)):
				infodict["poch"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1001) and (rpmhobjc[x][y][1] == 6)):
				infodict["vers"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1002) and (rpmhobjc[x][y][1] == 6)):
				infodict["rels"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1004) and (rpmhobjc[x][y][1] == 9)):
				infodict["summ"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1005) and (rpmhobjc[x][y][1] == 9)):
				infodict["desc"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1015) and (rpmhobjc[x][y][1] == 6)):
				infodict["pkgr"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1020) and (rpmhobjc[x][y][1] == 6)):
				infodict["urln"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1006) and (rpmhobjc[x][y][1] == 4)):
				infodict["RPMTAG_BUILDTIME"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1009) and (rpmhobjc[x][y][1] == 4)):
				infodict["RPMTAG_SIZE"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 4)):
				infodict["SIGTAG_PAYLOADSIZE"] = safestri(rpmhobjc[x][y][4])
			
			elif ((rpmhobjc[x][y][0] == 1014) and (rpmhobjc[x][y][1] == 6)):
				infodict["licn"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1011) and (rpmhobjc[x][y][1] == 6)):
				infodict["vndr"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1016) and (rpmhobjc[x][y][1] == 9)):
				infodict["grup"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 6)):
				infodict["host"] = safestri(rpmhobjc[x][y][4])
			elif ((rpmhobjc[x][y][0] == 1044) and (rpmhobjc[x][y][1] == 6)):
				infodict["srpm"] = safestri(rpmhobjc[x][y][4])
			
			elif ((rpmhobjc[x][y][0] == 1047) and (rpmhobjc[x][y][1] == 8)):
				infodict["prvn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1113) and (rpmhobjc[x][y][1] == 8)):
				infodict["prvl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1112) and (rpmhobjc[x][y][1] == 4)):
				infodict["prvf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1049) and (rpmhobjc[x][y][1] == 8)):
				infodict["reqn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1050) and (rpmhobjc[x][y][1] == 8)):
				infodict["reql"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1048) and (rpmhobjc[x][y][1] == 4)):
				infodict["reqf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1054) and (rpmhobjc[x][y][1] == 8)):
				infodict["conn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1055) and (rpmhobjc[x][y][1] == 8)):
				infodict["conl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1053) and (rpmhobjc[x][y][1] == 4)):
				infodict["conf"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1090) and (rpmhobjc[x][y][1] == 8)):
				infodict["obsn"] = safestri(rpmhobjc[x][y][4]).split("\n")
			elif ((rpmhobjc[x][y][0] == 1115) and (rpmhobjc[x][y][1] == 8)):
				infodict["obsl"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1114) and (rpmhobjc[x][y][1] == 4)):
				infodict["obsf"] = rpmhobjc[x][y][4].split("\n")
			
			elif ((rpmhobjc[x][y][0] == 1118) and (rpmhobjc[x][y][1] == 8)):
				infodict["flep"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1117) and (rpmhobjc[x][y][1] == 8)):
				infodict["flen"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1116) and (rpmhobjc[x][y][1] == 4)):
				infodict["flei"] = rpmhobjc[x][y][4].split("\n")
			elif ((rpmhobjc[x][y][0] == 1030) and (rpmhobjc[x][y][1] == 3)):
				infodict["flet"] = rpmhobjc[x][y][4].split("\n")
	
	return infodict

def procenvr(envrstri):
	outpstri = ""
	
	if (envrstri == ""):
		return outpstri
	
	verslist = envrstri.rsplit("-", 1)
	pochlist = verslist[0].split(":", 1)
	
	if (len(verslist) > 1):
		outpstri = ((" rel=\"%s\"" % (verslist[1])) + outpstri)
	
	if (len(pochlist) < 2):
		pochlist.insert(0, "0")
	
	outpstri = ((" epoch=\"%s\" ver=\"%s\"" % (pochlist[0], pochlist[1])) + outpstri)
	
	return outpstri

def rpmelist(namelist, verslist, flaglist):
	x = len(namelist); y = len(verslist); z = len(flaglist)
	d = {"2":"LT", "4":"GT", "8":"EQ", "10":"LE", "12":"GE"}
	o = ""
	
	if (x < 1):
		return o
	
	if ((x <= y) and (y <= z)):
		f = []
		for w in range(0, x):
			f.append(["", ""])
			
			n = str(int(flaglist[w]) & 0xf)
			try:
				f[w][0] = (" flags=\"%s\"" % (d[n]))
			except:
				pass
			
			n = ((int(flaglist[w]) & 0x200) + (int(flaglist[w]) & 0x100))
			if (n != 0):
				f[w][1] = (" pre=\"1\"")
		
		l = []
		for w in range(0, x):
			if (re.match("^.*rpmlib.*$", namelist[w], re.I)):
				continue
			
			i = procenvr(verslist[w])
			n = ("name=\"%s\"%s%s%s" % (namelist[w], f[w][0], i, f[w][1]))
			
			if (n in l):
				continue
			
			o += ("\t\t\t<rpm:entry %s/>\n" % (n))
			l.append(n)
	
	return o

def makexmld(rpmhdict):
	o = ""
	
	o += ("<package type=\"rpm\">\n")
	
	o += ("\t<name>%s</name>\n" % (rpmhdict["name"]))
	o += ("\t<arch>%s</arch>\n" % (rpmhdict["arch"]))
	o += ("\t<version epoch=\"%s\" ver=\"%s\" rel=\"%s\"/>\n" % (rpmhdict["poch"], rpmhdict["vers"], rpmhdict["rels"]))
	o += ("\t<checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n" % (rpmhdict["hash"]))
	o += ("\t<summary>%s</summary>\n" % (rpmhdict["summ"]))
	o += ("\t<description>%s</description>\n" % (rpmhdict["desc"]))
	o += ("\t<packager>%s</packager>\n" % (rpmhdict["pkgr"]))
	o += ("\t<url>%s</url>\n" % (rpmhdict["urln"]))
	o += ("\t<time file=\"%s\" build=\"%s\"/>\n" % (int(rpmhdict["RPMTAG_BUILDTIME"]) - 10, rpmhdict["RPMTAG_BUILDTIME"]))
	o += ("\t<size package=\"%s\" installed=\"%s\" archive=\"%s\"/>\n" % (rpmhdict["size"], rpmhdict["RPMTAG_SIZE"], rpmhdict["SIGTAG_PAYLOADSIZE"]))
	o += ("\t<location href=\"%s\"/>\n" % (rpmhdict["locn"]))
	
	o += ("\t<format>\n")
	o += ("\t\t<rpm:license>%s</rpm:license>\n" % (rpmhdict["licn"]))
	o += ("\t\t<rpm:vendor>%s</rpm:vendor>\n" % (rpmhdict["vndr"]))
	o += ("\t\t<rpm:group>%s</rpm:group>\n" % (rpmhdict["grup"]))
	o += ("\t\t<rpm:buildhost>%s</rpm:buildhost>\n" % (rpmhdict["host"]))
	o += ("\t\t<rpm:sourcerpm>%s</rpm:sourcerpm>\n" % (rpmhdict["srpm"]))
	o += ("\t\t<rpm:header-range start=\"%s\" end=\"%s\"/>\n" % (rpmhdict["heds"], rpmhdict["hede"]))
	
	try:
		t = rpmelist(rpmhdict["prvn"], rpmhdict["prvl"], rpmhdict["prvf"])
		o += ("\t\t<rpm:provides>\n%s\t\t</rpm:provides>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["reqn"], rpmhdict["reql"], rpmhdict["reqf"])
		o += ("\t\t<rpm:requires>\n%s\t\t</rpm:requires>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["conn"], rpmhdict["conl"], rpmhdict["conf"])
		o += ("\t\t<rpm:conflicts>\n%s\t\t</rpm:conflicts>\n" % (t))
	except:
		pass
	
	try:
		t = rpmelist(rpmhdict["obsn"], rpmhdict["obsl"], rpmhdict["obsf"])
		o += ("\t\t<rpm:obsoletes>\n%s\t\t</rpm:obsoletes>\n" % (t))
	except:
		pass
	
	try:
		x = len(rpmhdict["flen"])
		y = len(rpmhdict["flei"])
		z = len(rpmhdict["flet"])
	except:
		x = -1; y = -2; z = -3
	
	if ((x == y) and (y == z)):
		l = []
		for u in range(0, z):
			if (rpmhdict["flen"][u] == ""):
				break
			
			i = int(rpmhdict["flei"][u])
			pathname = (rpmhdict["flep"][i].rstrip("/") + "/" + rpmhdict["flen"][u])
			if (not re.match("^(/etc.*|.*bin/.*|/usr/lib/sendmail)$", pathname, re.I)):
				continue
			
			if (pathname in l):
				continue
			
			n = (int(rpmhdict["flet"][u]) & 0x4000)
			if (n != 0):
				o += ("\t\t<file type=\"dir\">%s</file>\n" % (pathname))
			
			else:
				o += ("\t\t<file>%s</file>\n" % (pathname))
			
			l.append(pathname)
	
	o += ("\t</format>\n")
	
	o += ("</package>\n")
	
	return [o]

def findfile(foldname):
	x = 0
	filelist = [foldname]
	remolist = []
	
	while (x < len(filelist)):
		if (os.path.isdir(filelist[x])):
			templist = os.listdir(filelist[x])
			
			for tempitem in templist:
				filelist.append(filelist[x] + "/" + tempitem)
			
			remolist.append(filelist[x])
		
		else:
			if (not re.match("^.*\.rpm$", filelist[x], re.I)):
				remolist.append(filelist[x])
			
			elif (re.match("^(%s/repodata/.*|.*debuginfo.*|.*\.src\.rpm)$" % (foldname), filelist[x], re.I)):
				remolist.append(filelist[x])
		
		x += 1
	
	for remoitem in remolist:
		while (filelist.count(remoitem) > 0):
			filelist.remove(remoitem)
	
	return filelist

def main():
	if (len(sys.argv) < 2):
		print("Usage: %s <repo-fold>" % (sys.argv[0]))
		sys.exit(0)
	
	repofold = sys.argv[1].rstrip("/")
	repolist = findfile(repofold)
	repoleng = len(repolist)
	
	print("found [%d] rpm files..." % (repoleng))
	
	childnum = 3
	destfold = "/tmp/repodata"
	pipelist = []; proclist = []
	
	try:
		os.mkdir(destfold)
	except:
		pass
	
	print("beg primary.xml...")
	
	for x in range(0, childnum):
		(parnpipe, chilpipe) = multiprocessing.Pipe(True)
		procnumb = os.fork()
		
		if (procnumb == 0):
			parnpipe.close()
			fileobjc = open("%s/primary.%d.xml" % (destfold, x), "w")
			
			while (1):
				chilpipe.send("more")
				filename = chilpipe.recv()
				
				if (not filename):
					break
				
				print("[%d] : %s" % (x, filename))
				headobjc = rpmhinfo(repofold + "/", filename)
				
				try:
					headdict = makedict(headobjc)
					xmldobjc = makexmld(headdict)
					fileobjc.write(xmldobjc[0])
				except:
					sys.stderr.write("erro:" + str(sys.exc_info()) + "\n")
					pass
			
			fileobjc.close()
			sys.exit(0)
		
		else:
			chilpipe.close()
			pipelist.append(parnpipe)
			proclist.append(procnumb)
	
	while (len(pipelist) > 0):
		(readlist, outplist, errolist) = select.select(pipelist, [], [])
		
		for pipeitem in pipelist:
			if (pipeitem not in readlist):
				continue
			
			try:
				dumydata = pipeitem.recv()
				
				if (len(repolist) > 0):
					pipeitem.send(repolist[0])
					repolist.pop(0)
				else:
					pipeitem.send("")
			
			except:
				pipelist.remove(pipeitem)
	
	for procitem in proclist:
		try:
			os.waitpid(procitem, 0)
		except:
			pass
	
	print("end primary.xml...")
	
	primoutp = ""
	for x in range(0, childnum):
		try:
			fileobjc = open("%s/primary.%d.xml" % (destfold, x), "r")
			primoutp += fileobjc.read()
			fileobjc.close()
			os.unlink("%s/primary.%d.xml" % (destfold, x))
		except:
			pass
	
	fileobjc = open("%s/primary.xml" % (destfold), "w")
	fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
	fileobjc.write("<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n" % (repoleng))
	fileobjc.write(primoutp)
	fileobjc.write("</metadata>\n")
	fileobjc.close()
	
	fileobjc = open("%s/repomd.xml" % (destfold), "w")
	fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
	fileobjc.write("<repomd xmlns=\"http://linux.duke.edu/metadata/repo\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\">\n")
	fileobjc.write("<data type=\"primary\">\n")
	fileobjc.write("\t<checksum type=\"sha256\">%s</checksum>\n" % (hashlib.sha256(primoutp).hexdigest()))
	fileobjc.write("\t<location href=\"repodata/primary.xml\"/>\n")
	fileobjc.write("\t<size>%d</size>\n" % (len(primoutp)))
	fileobjc.write("</data>\n")
	fileobjc.write("</repomd>\n")
	fileobjc.close()

if (__name__ == "__main__"):
	main()

createrepo From Scratch Using Forks & Pipes

Hijacking The Mock Yum Buildroot

So I wanted to see if I could speed up common mock tasks without using their built-in caching methods as the buildroot package set is changing and it may not pick up those changes. The mock source code is located here (https://fedorahosted.org/mock/) and the results below show that on an x64 machine, the nled build went 22.5% faster! The downside is that you would have to set aside a dedicate builder to constantly create and tar the updated buildroots for the builders running the modified mock. Also note that the speed-up shown for the modified mock did not have to download the tar as it was locally stored.

Regular mock

# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time mock -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm 
INFO: mock.py version 1.1.9 starting...
...
INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 1 minutes 36 seconds
...
DEBUG: kill orphans

real	1m37.880s
user	1m3.519s
sys	0m4.444s

Modified mock

# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time ./py/mock.py -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm 
INFO: mock.py version unreleased_version starting...
...
INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 0 minutes 21 seconds
...
DEBUG: kill orphans

real	0m22.001s
user	0m44.526s
sys	0m2.925s

./py/mockbuild/backend.py

155:    decorate(traceLog())
156:    def _unlock_and_rm_chroot(self):
157:        if (os.path.exists("/tmp/buildroot.flag")):
158:            print("CUSTOM: HIJACK SAVING [%s]" % (self.makeChrootPath()))
159:            os.system("tar -cf \"/tmp/buildroot.tar\" -C '%s' ." % (self.makeChrootPath()))
160:            try:
161:                os.unlink("/tmp/buildroot.flag")
162:            except:
163:                pass
...
826:    decorate(traceLog())
827:    def _yum(self, cmd, returnOutput=0):
828:        """use yum to install packages/package groups into the chroot"""
829:        flag=0
830:        try:
831:            if ((cmd[0] == "groupinstall") and (cmd[1] == "buildsys-build")):
832:                flag=1
833:                if (os.path.exists("/tmp/buildroot.tar")):
834:                    os.system("tar -xvf /tmp/buildroot.tar -C '%s'" % (self.makeChrootPath()))
835:                    return ""
836:        except:
837:            pass
...
853:        try:
854:            self._callHooks("preyum")
855:            output = mockbuild.util.do(yumcmd, returnOutput=returnOutput)
856:            self._callHooks("postyum")
857:            if (flag == 1):
858:                os.system("touch /tmp/buildroot.flag")
859:                raise mockbuild.exception.YumError, "CUSTOM: HIJACK YUM ERROR"
860:            else:
861:                return output
862:        except mockbuild.exception.Error, e:
863:            raise mockbuild.exception.YumError, str(e)
Hijacking The Mock Yum Buildroot

Repo Management Helper Script

So we’re finally on to building Fedora 15 for ARMv5 on F13 builders and since we’re managing a repo manually, I needed a script to do the following:

  • Move all failed build packages back to the source folder to be rebuilt again
  • Move all completed build packages not in the destination to that directory
  • Move any build packages in the destination that are not built back to the source folder

Here is the following script which helps us achieve this:

fixs.py

import os
import re
import subprocess

z = '/var/export/f15v5.0'
os.system("mkdir \"%s/fixs/\" 2> /dev/null" % (z))

r = subprocess.Popen(["find", "%s/repo" % (z), "-type", "f"], stdout=subprocess.PIPE).communicate()[0]
s = r.split("\n")

f = subprocess.Popen(["ls", "%s/dest" % (z)], stdout=subprocess.PIPE).communicate()[0]
g = f.split("\n")

for i in g:
	i = i.strip()
	n = re.sub("\.src\.rpm$", "", i)
	m = [0, 0]
	for j in s:
		j = j.strip()
		k = re.sub("[^0-9A-Za-z]", ".", n)
		if (re.match("^.*%s.*$" % (k), j)):
			m[0] = 1
		if (re.match("^.*%s\.src\.rpm$" % (k), j)):
			m[1] = 1
	c = ("mv \"%s/dest/%s\" \"%s/fixs/\"" % (z, i, z))
	if ((m[0] == 1) and (m[1] == 0)):
		print(c)
		os.system(c)
	if ((m[0] == 0) and (m[1] == 0)):
		print(c)
		os.system(c)
Repo Management Helper Script

Mini-Koji Build System (Moji)

This was a small project used to help with the initial boot-strapping and rebuild of the various Fedora-ARM packages without reliance on previous Fedora builds. We were unable to do this with Koji because it only lets you build a single version of a package once. This was a problem because we used F13 builders to build F15 and then we upgrade the builders to F15 so that they could rebuild F15 once more. This was done to help produce the cleanest and most native possible build that we could for F15 for both ARMv5 and ARMv7.

Edit: I also did a little Koji mod which could bypass the “one-build” restriction here

Source Code (Git Hub)
Binary Program (Fedora People)

Mini-Koji Build System (Moji)

Fedora ARM On A Panda From Scratch

Below is a script to help create a panda sdcard from scratch but using pre-compiled kernel and boot files.

#!/bin/bash

if [ "$1" == "" ]
then
	echo "Usage: $0 <device>"
	exit 1
fi

# http://www.angstrom-distribution.org/demo/pandaboard/mkcard.txt

export CARD="$1"
export BOOT="/mnt/boot"
export ROOT="/mnt/root"
export TEMP="/tmp/temp"

echo "formatting [${CARD}] after sleeping..." ; sleep 10
mkdir ${TEMP} 2> /dev/null
umount ${CARD}{1..4} 2> /dev/null
dd if=/dev/zero of=${CARD} bs=1024 count=1024
./mkcard.sh ${CARD}

# http://www.angstrom-distribution.org/demo/pandaboard/{MLO,u-boot.bin,uImage}

umount ${BOOT} 2> /dev/null ; mkdir ${BOOT} 2> /dev/null ; mount ${CARD}1 ${BOOT}

wget -O ${BOOT}/MLO 'http://www.angstrom-distribution.org/demo/pandaboard/MLO' ; sync
wget -O ${BOOT}/u-boot.bin 'http://www.angstrom-distribution.org/demo/pandaboard/u-boot.bin'
wget -O ${BOOT}/uImage 'http://www.angstrom-distribution.org/demo/pandaboard/uImage'

time sync ; umount ${BOOT} 2> /dev/null

# http://fedoraproject.org/wiki/Architectures/ARM
# http://scotland.proximity.on.ca/fedora-arm/rootfs/f13-rc1/rootfs-f13-rc1-2011-06-29.tar.bz2

umount ${ROOT} 2> /dev/null ; mkdir ${ROOT} 2> /dev/null ; mount ${CARD}2 ${ROOT}

tar -xjvf rootfs-f*.tar.bz2 -C ${ROOT} ; mv ${ROOT}/rootfs-*/* ${ROOT} 2> /dev/null

time sync ; umount ${ROOT} 2> /dev/null

# path to kernel modules

umount ${ROOT} 2> /dev/null ; mkdir ${ROOT} 2> /dev/null ; mount ${CARD}2 ${ROOT}

echo '# custom added section' >> ${ROOT}/etc/rc.local
echo '/sbin/ifconfig usb0 inet 192.168.1.253 netmask 255.255.255.0 up' >> ${ROOT}/etc/rc.local
echo '/sbin/route add default gw 192.168.1.254' >> ${ROOT}/etc/rc.local
echo 'while true ; do /sbin/agetty -L 115200 console vt100 ; done' >> ${ROOT}/etc/rc.local

time sync ; umount ${ROOT} 2> /dev/null

# done

eject ${CARD}

Note: A good guide to compiling the kernel is located here: http://wiki.meego.com/ARM/OMAP4_Panda

Fedora ARM On A Panda From Scratch

Basic Stats Page Outside Of Koji

So since we’re building Fedora ARM packages outside of Koji first (on F13 builders and then again on F15 builders), I decided to create a simple stats page so our team could track its status.

<?php
	function fileread($filename)
	{
		$outpstri = "";
		$fileobjc = fopen($filename, "r");
		$outpstri .= fread($fileobjc, filesize($filename));
		fclose($fileobjc);
		$outpstri .= "\n";
		return $outpstri;
	}
	$foldname = "./done";
	if (isset($_GET["pkgn"]))
	{
		$outpstri = "";
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/build.log");
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/root.log");
		$outpstri .= fileread($foldname."/mock.".$_GET["pkgn"]."/state.log");
		$outpstri = preg_replace("/&/", "&amp;", $outpstri);
		$outpstri = preg_replace("/</", "&lt;", $outpstri);
		$outpstri = preg_replace("/>/", "&gt;", $outpstri);
		$outpstri = str_replace("\n", " <br /> \n ", $outpstri);
		print($outpstri."\n");
		die;
	}
	exec("find ".$foldname." -type f", $outplist);
	$donestri = ""; $donelist = array();
	$errostri = ""; $errolist = array();
	$totllist = array();
	foreach ($outplist as $outpitem)
	{
		$safefold = preg_replace("/([\/.])/", "\\\\$1", $foldname);
		if (!preg_match("/^".$safefold."\/mock\.([^\/]+)\/.*$/", $outpitem))
		{
			continue;
		}
		$pkgsname = preg_replace("/^".$safefold."\/mock\.([^\/]+)\/.*$/", "$1", $outpitem);
		if (preg_match("/^.*\.src\.rpm$/", $outpitem))
		{
			if (!in_array($pkgsname, $donelist))
			{
				array_push($donelist, $pkgsname);
			}
		}
		if (!in_array($pkgsname, $totllist))
		{
			array_push($totllist, $pkgsname);
		}
	}
	foreach ($totllist as $totlitem)
	{
		if (!in_array($totlitem, $donelist))
		{
			array_push($errolist, $totlitem);
		}
	}
	sort($donelist);
	sort($errolist);
	$donenumb = count($donelist);
	$erronumb = count($errolist);
	$donestri .= ("<script>parent.document.title += ' [ ".$donenumb.".d ] ';</script>");
	$donestri .= ("<b>Built: [ ".$donenumb." ]</b> <br /> <br /> \n");
	foreach ($donelist as $doneitem)
	{
		$donestri .= ($doneitem." <br /> \n ");
	}
	$errostri .= ("<script>parent.document.title += ' [ ".$erronumb.".e ] ';</script>");
	$errostri .= ("<b>Error: [ ".$erronumb." ]</b> <br /> <br /> \n");
	foreach ($errolist as $erroitem)
	{
		$errostri .= ("<a href=\"javascript:function a() { parent.document.getElementById('logswind').src = ('".$_SERVER["PHP_SELF"]."?pkgn=".$erroitem."'); } a();\">".$erroitem."</a> <br /> \n ");
	}
	if (isset($_GET["done"]))
	{
		print($donestri);
		die;
	}
	if (isset($_GET["erro"]))
	{
		print($errostri);
		die;
	}
?>
<html>
	<head>
		<title>loading...</title>
		<style>
			table
			{
				width: 95%;
			}
			iframe
			{
				border: 1px dotted black;
				width: 100%;
			}
			#errowind
			{
				height: 160px;
			}
			#logswind
			{
				height: 240px;
			}
			#donewind
			{
				height: 160px;
			}
		</style>
		<script>
			function pkgslist()
			{
				document.title = "";
				document.getElementById("errowind").src = ('<?php echo $_SERVER["PHP_SELF"]; ?>?erro=true');
				document.getElementById("donewind").src = ('<?php echo $_SERVER["PHP_SELF"]; ?>?done=true');
			}
			function pageinit()
			{
				setInterval("pkgslist();", 30 * 1000);
			}
		</script>
	</head>
	<body onload="pkgslist(); pageinit();">
		<b>Date: [ <?php echo date("y-m-d H:i:s"); ?> ]</b>
		<br /> <br />
		<center>
		<table>
			<tr>
				<td><iframe id="errowind"></iframe></td>
			</tr>
			<tr>
				<td> &nbsp; </td>
			</tr>
			<tr>
				<td><b>Logs:</b></td>
			</tr>
			<tr>
				<td><iframe id="logswind"></iframe></td>
			</tr>
			<tr>
				<td> &nbsp; </td>
			</tr>
			<tr>
				<td><iframe id="donewind"></iframe></td>
			</tr>
		</table>
		</center>
	</body>
</html>

Basic Stats Page Outside Of Koji

F15 ARMv5 Building On F13 Builders

This script can be distributed to multiple builders to build the various Fedora packages with mock. The setup commands for the server side are included (but commented) in the script below. Note: The script now requires an apache server to be running which allows for the new fedora-release package information to be retrieved for correct build tagging.

> httpd.conf

Alias /builds/ "/tmp/builds/"

<Directory "/tmp/builds">
    Options Indexes MultiViews FollowSymLinks
    AllowOverride None
    Order allow,deny
    Allow from all
</Directory>

> arm-builder.sh

#!/bin/bash

# COMMON: variable definitions for this script

SSHDHOST="1.2.3.4"
SSHDUSER="builder"
SSHDPASS="/tmp/sshkey"
SSHDROOT="/tmp/builds"
LOCLTEMP="/tmp/work"

SSHDPRIO="${SSHDROOT}/prio"
SSHDPKGS="${SSHDROOT}/pkgs"
SSHDWORK="${SSHDROOT}/work"
SSHDDONE="${SSHDROOT}/done"

# SERVER: local and remote account setup

#passwd ; chsh -s /bin/bash
#mkdir -p ~/.ssh ; echo 'ssh-pub-key@local' > ~/.ssh/authorized_keys
#chmod 700 ~/.ssh ~/.ssh/authorized_keys

# SERVER: build directory creation

#mkdir -p ${SSHDPRIO} ${SSHDPKGS} ${SSHDWORK} ${SSHDDONE}

# SERVER: create intial build package files

#koji list-tagged dist-f15 --latest > /tmp/list.txt
#cd ${SSHDPKGS} ; tail -n +3 /tmp/list.txt | awk '{ print $1 }' | while read line ; do touch "$line" ; done

# SERVER: initialize the most basic repo containing F15 release information

#urls="http://kojipkgs.fedoraproject.org/packages/fedora-release/15/1"
#cd ${SSHDDONE} ; wget "${urls}/src/fedora-release-15-1.src.rpm" "${urls}/noarch/fedora-release-15-1.noarch.rpm" "${urls}/noarch/fedora-release-rawhide-15-1.noarch.rpm" ; createrepo --update -d ${SSHDDONE}

# CLIENT: save the ssh key for auto logins

(echo '-----BEGIN ssh-pri-key-----') > ${SSHDPASS}
chmod 700 ${SSHDPASS}

# CLIENT: create a custom fedora repo file for the mock builds

echo "
config_opts['root'] = 'fedora-15-arm'
config_opts['target_arch'] = 'armv5tel'
config_opts['legal_host_arches'] = ('armv5tel', 'armv6l', 'armv7l')
config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
config_opts['dist'] = 'fc15'
config_opts['plugin_conf']['root_cache_enable'] = False
config_opts['plugin_conf']['yum_cache_enable'] = False

config_opts['yum.conf'] = \"\"\"
[main]
cachedir=/var/cache/yum
debuglevel=1
reposdir=/dev/null
logfile=/var/log/yum.log
retries=20
obsoletes=1
gpgcheck=0
assumeyes=1
syslog_ident=mock
syslog_device=

# repos

[custom]
name=fedora-custom
baseurl=http://${SSHDHOST}/builds/done/
enabled=1

### NOTE : USE A VAR HERE FOR THE PATH INSTEAD ###

[f15-arm-koji]
name=fedora15-arm-koji
baseurl=http://arm.koji.fedoraproject.org/repos/dist-f15-build/latest/arm/
enabled=1

[fedora]
name=fedora
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=fedora-13&arch=arm
failovermethod=priority

[updates-released]
name=updates
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f13&arch=arm
failovermethod=priority
\"\"\"
" > /etc/mock/fedora-15-arm.cfg
chmod 755 /etc/mock/fedora-15-arm.cfg

# CLIENT: create a tmp working directory for the pkg builds

mkdir -p ${LOCLTEMP}
chmod 777 ${LOCLTEMP}
cd ${LOCLTEMP}

# CLIENT: ensure that the mock user is in the mock group (not just primary)

useradd -g mock mock
usermod -G mock mock
chsh -s /bin/bash mock

# CLIENT: loop to keep building pkgs

while true
do
	# CLIENT: check for a random pkg and move it out of the directory so it can be worked on
	
	for dirn in ${SSHDPRIO} ${SSHDPKGS}
	do
		pkgn=$(ssh -i ${SSHDPASS} ${SSHDUSER}@${SSHDHOST} "pkgn=\$(ls -t ${dirn} | tail -n 1) ; if [ \"\${pkgn}\" != \"\" ] ; then mv ${dirn}/\${pkgn} ${SSHDWORK}/ ; echo \${pkgn} ; fi")
		
		if [ "${pkgn}" != "" ]
		then
			break
		fi
	done
	
	# CLIENT: remove any old build files
	
	rm -frv ${LOCLTEMP}/*
	
	# CLIENT: if we have a pkg then try to download the source
	
	if [ "${pkgn}" != "" ]
	then
		# CLIENT: download and build the package
		
		echo "Building [${dirn}/${pkgn}] in [`pwd`]"
		
		# CLIENT: download the source rpm pkg from the server
		
		scp -i ${SSHDPASS} ${SSHDUSER}@${SSHDHOST}:${SSHDWORK}/${pkgn} ./
		
		if [ ! -s "${pkgn}" ]
		then
			# CLIENT: pkg prep
			
			rm -frv ${LOCLTEMP}/*
			pkgn=$(echo "${pkgn}" | sed -e 's/\.src\.rpm$//g')
			
			#koji download-build --arch=src "${pkgn}"
			urln=`python -c "import rpmUtils.miscutils ; (n,v,r,e,a)=rpmUtils.miscutils.splitFilename('${pkgn}') ; print('http://kojipkgs.fedoraproject.org/packages/%s/%s/%s.%s/src/${pkgn}.src.rpm' % (n,v,r,a))"`
			wget "${urln}"
		fi
	fi
	
	# CLIENT: check for and make any build files readable
	
	file=$(echo *)
	
	# CLIENT: if we have a source pkg then try to build it
	
	if [ -e "${file}" ]
	then
		# CLIENT: pkg prep
		
		chmod 755 *
		
		# CLIENT: build the source rpm in mock
		
		su -c "mock -vr fedora-15-arm --resultdir=${LOCLTEMP}/mock.${pkgn} ${LOCLTEMP}/${file}" - mock
		
		# CLIENT: copy any mock results back to the server
		
		scp -i ${SSHDPASS} -r mock.${pkgn} ${SSHDUSER}@${SSHDHOST}:${SSHDDONE}/
	fi
	
	sleep 60
done

> repo.py

import os
import sys
import time

repo = "/var/export/f15v5.0/repo"
temp = "/var/export/f15v5.0/temp"

def syse(comd, show=True):
	if (show):
		print(comd)
	
	os.system(comd)

while (1):
	syse("find %s -type f -iname '*.[rx][pm][ml]' -exec ls --time-style='+%%s' -l {} \; 2> /dev/null | tr -s ' ' | cut -d ' ' -f 6- | sort -nr | head -n 1 | grep -i '.*\.rpm$' > /tmp/repo.txt" % (repo))
	
	fobj = open("/tmp/repo.txt", "r")
	chek = fobj.read().strip()
	fobj.close()
	
	if (chek != ""):
		syse("echo \"Starting on [ `date` ]\" | tee -a ~/repo.log")
		
		syse("mkdir \"%s\" 2> /dev/null" % (temp)) # make sure the temp repo directory exists
		syse("rm -fr %s/* 2> /dev/null" % (temp)) # clear the temp repo directory
		
		for item in os.listdir(repo):
			syse("ln -s \"%s/%s\" \"%s/%s\"" % (repo, item, temp, item), show=False)
		
		syse("rm -fr \"%s/repodata\"" % (temp)) # remove any temp repo data links
		syse("cp -fr \"%s/repodata\" \"%s/\"" % (repo, temp)) # copy over the current repo data
		syse("createrepo --update -d \"%s\" --exclude=*debuginfo* --exclude=*.src.rpm" % (temp)) # create/update the new repo data
		syse("rm -fr \"%s/repodata\"" % (repo)) # remove the old repodata files
		syse("cp -fr \"%s/repodata\" \"%s/\"" % (temp, repo)) # copy over the new repo data
		
		syse("echo \"Ending on [ `date` ]\" | tee -a ~/repo.log")
	
	time.sleep(60) # sleep loop or risk fire!
F15 ARMv5 Building On F13 Builders

Wikipedia Monitoring IRC Bot

Sep 03 06:21:19 <wikimon> [ http://zenit.senecac.on.ca/wiki/index.php?title=CDOT& ] was changed by [ ShaneStrauss ] on [ 06:18, 3 September 2011 ]
Sep 03 09:11:19 <wikimon> [ http://zenit.senecac.on.ca/wiki/index.php?title=CDOT& ] was changed by [ Fossjon ] on [ 09:08, 3 September 2011 ]

#!/usr/bin/python

import os
import re
import select
import socket
import sys
import time
import urllib2

def listfilt(listobjc, filtstri, onceonly=0):
	outpflag = 0
	outplist = []
	
	for listitem in listobjc:
		if (re.match("^" + filtstri + "$", listitem)):
			outplist.append(listitem)
			
			if (onceonly == 1):
				outpflag = 1
		
		elif (outpflag == 1):
			outplist.append(listitem)
	
	return outplist

def getspage(urlnstri):
	headitem = ""
	
	try:
		websobjc = urllib2.urlopen(urlnstri, None, 10)
		dataread = websobjc.read().replace("\t", "").replace("\r", "").replace("\n", "")
		
		listlist = dataread.replace("<li", "\n<li").replace("</li", "</li\n").split("\n")
		listlist = listfilt(listlist, "<li.*checked.*")
		
		linklist = re.sub("(<a [^>]+>[^<]+</a>)", "\n\\1\n", listlist[0]).split("\n")
		linklist = listfilt(linklist, "<a [^>]+>[^<]+</a>.*")
		
		pagestri = re.sub("action=history[&]*", "", urlnstri).strip()
		datestri = re.sub("<[^>]+>", "", linklist[1]).strip()
		namestri = re.sub("<[^>]+>", "", linklist[2]).strip()
		
		headitem = ("[ %s ] was changed by [ %s ] on [ %s ]" % (pagestri, namestri, datestri))
	
	except KeyboardInterrupt:
		sys.exit(0)
	
	except:
		pass
	
	return headitem

sendline_last = 0; sendline_list = []

def sendline(circobjc, sendstri):
	global sendline_last
	global sendline_list
	
	prestime = time.time()
	tempstri = sendstri.strip()
	
	if (tempstri != ""):
		sendline_list.append(tempstri)
	
	if (((prestime - sendline_last) >= 1) and (len(sendline_list) > 0)):
		print("[SEND] %s" % (sendline_list[0]))
		circobjc.send(sendline_list[0] + "\r\n")
		
		sendline_list.pop(0)
		sendline_last = prestime

readline_data = ""

def readline(circobjc):
	global readline_data
	
	outpstri = ""
	(readlist, outplist, errolist) = select.select([circobjc], [], [], 0)
	
	if (circobjc in readlist):
		tempstri = circobjc.recv(2**20)
		
		if (not tempstri):
			readline_data = ""
			return None
		
		readline_data += tempstri
	
	try:
		newlindx = readline_data.index("\n")
	
	except:
		newlindx = -1
	
	if (newlindx != -1):
		outpstri = readline_data[:newlindx].strip()
		newlindx += 1
		readline_data = readline_data[newlindx:]
	
	return outpstri

procirco_last = 0

def procirco(circobjc, channame, lastmesg):
	global procirco_last
	
	prestime = time.time()
	
	if ((prestime - procirco_last) >= 60):
		sendline(circobjc, "JOIN " + channame)
		procirco_last = prestime
	
	regxobjc = re.match("^PING (.*)$", lastmesg)
	
	if (regxobjc):
		sendline(circobjc, "PONG " + regxobjc.group(1))

def newcirco(hostname, nickname):
	circobjc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	
	circobjc.connect((hostname, 6667))
	sendline(circobjc, "USER Alex * * : Bob")
	sendline(circobjc, "NICK %s" % (nickname))
	
	return circobjc

def main():
	if (len(sys.argv) < 4):
		print("Usage: %s <host> <nick> <chan>" % (sys.argv[0]))
		sys.exit(0)
	
	hostname = sys.argv[1]; nickname = sys.argv[2]; channame = sys.argv[3]
	pagelist = []; lastlist = []; lasttime = 0
	circobjc = None; circline = ""; circflag = 0
	
	while (1):
		circflag = 0
		prestime = time.time()
		
		if (circobjc == None):
			circobjc = newcirco(hostname, nickname)
		
		else:
			circline = readline(circobjc)
			
			if (circline == None):
				circobjc = None
			
			else:
				circflag = 1
				
				if (circline != ""):
					print("[RECV] %s" % (circline))
				
				procirco(circobjc, channame, circline)
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s add (.*)$" % (nickname), circline, re.I)
				
				if (regxobjc):
					if (regxobjc.group(2) not in pagelist):
						pagelist.append(regxobjc.group(2))
						lastlist.append("")
						lasttime = 0
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s del (.*)$" % (nickname), circline, re.I)
				
				if (regxobjc):
					for x in range(len(pagelist) - 1, -1, -1):
						try:
							if (re.match(regxobjc.group(2), pagelist[x], re.I)):
								sendline(circobjc, "PRIVMSG %s :deleting [ %s ]" % (regxobjc.group(1), pagelist[x]))
								lastlist.pop(x)
								pagelist.pop(x)
						
						except:
							pass
				
				regxobjc = re.match("^:([^!]+)![^ ]+ privmsg [^ ]+ :%s out.*$" % (nickname), circline, re.I)
				
				if (regxobjc):
					for listitem in lastlist:
						sendline(circobjc, "PRIVMSG %s :%s" % (regxobjc.group(1), listitem))
			
			if ((prestime - lasttime) >= (5 * 60)):
				for x in range(0, len(pagelist)):
					tempoutp = getspage(pagelist[x])
					
					if ((tempoutp != "") and (tempoutp != lastlist[x])):
						if (lastlist[x] != ""):
							sendline(circobjc, "PRIVMSG %s :%s" % (channame, tempoutp))
						
						lastlist[x] = tempoutp
				
				lasttime = prestime
		
		sendline(circobjc, "")
		
		if (circflag == 0):
			time.sleep(1)

if (__name__ == "__main__"):
	main()

Wikipedia Monitoring IRC Bot

Start Of The Semester

I just wanted to document a little more on what is pointing to what on our builder file systems. I put the commands and output below so hopefully one (or I) can follow it in the future in case file permissions or links get messed up.

# cat /etc/fstab

/dev/sdb1               /                       ext3    defaults             1 1
devpts                  /dev/pts                devpts  gid=5,mode=620       0 0
tmpfs                   /dev/shm                tmpfs   defaults             0 0
proc                    /proc                   proc    defaults             0 0
sysfs                   /sys                    sysfs   defaults             0 0
/hkexport/fs0.ext3      /fs0                    ext3    defaults,noauto,loop 0 0
/hkexport/swap0.ext3	/swap0                  ext3    defaults,noauto,loop 0 0
hongkong:/mnt/koji      /mnt/koji               nfs     defaults,noauto      0 0

# mount

/dev/sdb1 on / type ext3 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
australia:/var/export/cdot-guru-1-2 on /hkexport type nfs (rw,addr=192.168.1.196)
/dev/loop0 on /fs0 type ext3 (rw)
/dev/loop1 on /swap0 type ext3 (rw)
hongkong:/mnt/koji on /mnt/koji type nfs (rw,addr=192.168.1.254)

# ls -l /redirect

lrwxrwxrwx 1 root root 4 2010-11-05 13:34 /redirect -> /fs0

# for foldname in bin dev etc lib opt sys tmp usr var ; do find /${foldname} -type l -exec ls -l {} \; 2> /dev/null | grep -Ei ‘\-> /(fs0|redirect|hkexport|mnt/koji)’ ; done

lrwxrwxrwx 1 root root 23 2010-11-05 13:34 /var/lib/mock -> /redirect/mock/lib/mock
lrwxrwxrwx 1 root root 4 2010-11-05 13:34 /var/redirect -> /fs0
lrwxrwxrwx 1 root root 25 2010-11-05 13:34 /var/cache/mock -> /redirect/mock/cache/mock
Start Of The Semester