User:Millbot-Stats/genstats.py

Note that program is not licensed under GFDL, but under AGPL!

#!/usr/bin/python
# -*- coding: utf-8 -*-
# 
# Millbot-Stats, v. 1.1. A bot for generating statistics at MediaWiki sites.
# Copyright (C) 2008 Milos Rancic <millosh@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
import os
import sys
import time
import pickle
import stats
mydir = "./"
sys.path.append(mydir)
sys.path.append(mydir + "pywikipedia/")
from os.path import *
from wikipedia import *
 
#languages['ar'] = {
#       'name': "Arabic",
#       'full name': "Arabic language",
#       }
#projects['wikipedia'] = {
#       'baseurl': "wikipedia.org",
#       'lang addition type': "prefix",
#       'suffix': "/wiki/",
#       'langs': [ 'ar', '...', ],
#       'template': 'Template:Wikipedia statistics',  
#       }
 
projects = pickle.load(open("projects.pickle"))
languages = pickle.load(open("languages.pickle"))
 
transfile = mydir + "translations.conf.py"
execfile(transfile)
 
for plang in languages:
        if plang not in translations:
                translations[plang] = {}
        for slang in languages:
                if slang not in translations[plang]:
                        translations[plang][slang] = languages[slang]['full name']
                if languages[slang]['name'] not in translations[plang]:
                        translations[plang][languages[slang]['name']] = languages[slang]['name']
 
datadir = mydir + "data/"
 
#wikipedia_project = projects['wikipedia']
##projects = { 'wikiversity': projects['wikiversity'] }
pr_dictwikis = {}
pr_totals = {}
pr_listwikis = {}
truefalse = {}
 
for fam in projects:
        if fam not in truefalse:
                truefalse[fam] = {}
        project = projects[fam]
        langs = project['langs']
 
        listwikis = []
        dictwikis = {}
 
        totalgood = 0
        totaltotal = 0
        totaledits = 0
        totaladmins = 0
        totalusers = 0
        totalimages = 0
        oyear = time.strftime("%Y")
        omont = time.strftime("%m")
        odate = time.strftime("%d")
        ohour = time.strftime("%H")
        ominu = time.strftime("%M")
 
        for lang in langs:
                if project['lang addition type'] == 'prefix':
                        url = "http://" + lang + "." + project['baseurl'] + project['suffix'] + "Special:Statistics?action=raw"
                # else: ... # define your own types
                daydir = datadir + oyear + "/" + omont + "/" + odate + "/"
                odir = daydir + fam + "/" + lang + "/"
                if not isdir(odir):
                        cmd = "mkdir -p " + odir
                        os.system(cmd)
                fd = odir + "raw-stats-" + ohour + "-" + ominu + ".txt"
                command = "wget -O " + fd + " " + url
                os.system(command)
                row = file(fd).read()[:-1]
                if len(row) > 0:
                        cols = re.split(";",row)
                        good = re.sub("^.*?;good=([0-9]+);.*?$","\g<1>",row)
                        total = re.sub("^total=([0-9]+);.*?$","\g<1>",row)
                        edits = re.sub("^.*?;edits=([0-9]+);.*?$","\g<1>",row)
                        admins = re.sub("^.*?;admins=([0-9]+);.*?$","\g<1>",row)
                        users = re.sub("^.*?;users=([0-9]+);.*?$","\g<1>",row)
                        images = re.sub("^.*?;images=([0-9]+);.*?$","\g<1>",row)
                        index = float(good)
                        while index in dictwikis:
                                index -= 0.001
                        listwikis.append(index)
                        dictwikis[index] = {
                                'true': 'true',
                                'code': lang,
                                'good': good,
                                'total': total,
                                'edits': edits,
                                'admins': admins,
                                'users': users,
                                'images': images, 
                                'time': time.strftime("%Y-%m-%d %H:%M:%S"), 
                                }
                        totalgood += int(good)
                        totaltotal += int(total)
                        totaledits += int(edits)
                        totaladmins += int(admins)
                        totalusers += int(users)
                        totalimages += int(images)
                        truefalse[fam][lang] = "true"
                else:
                        truefalse[fam][lang] = "false"
        totals = {
                'totalgood': totalgood,
                'totaltotal': totaltotal,
                'totaledits': totaledits,
                'totaladmins': totaladmins,
                'totalusers': totalusers,
                'totalimages': totalimages,
                'totaltime': time.strftime("%Y-%m-%d %H:%M:%S")
                }
        listwikis.sort()
        listwikis.reverse()
        pr_dictwikis[fam] = dictwikis
        pr_totals[fam] = totals
        pr_listwikis[fam] = listwikis
 
stats_sites = {}
stats_sites['wikipedia'] = {}
stats_sites['wikipedia']['sr'] = {
        'projects': [
                'wikipedia', 'wiktionary', 'wikibooks', 'wikinews',
                'wikisource', 'wikiversity', 'wikiquote',
                ], 
        }
stats_sites['wiktionary'] = {}
stats_sites['wikibooks'] = {}
stats_sites['wikinews'] = {}
stats_sites['wikisource'] = {}
stats_sites['wikiversity'] = {}
stats_sites['wikiquote'] = {}
 
for st in stats_sites:
        sites = stats_sites[st]
        for s in sites:
                for fam in sites[s]['projects']:
                        dictwikis = pr_dictwikis[fam]
                        totals = pr_totals[fam]
                        listwikis = pr_listwikis[fam]
                        outfile = daydir + fam + "/stats-" + '-for-' + st + "-" + s + ".txt"
                        stats.engine(projects[st]['baseurl'],mydir,outfile,s,listwikis,dictwikis,translations,totals,sites)
                        #stats.engine(wikipedia_project['baseurl'],mydir,outfile,s,listwikis,dictwikis,translations,totals,sites)
                        content = file(outfile).read().decode('utf-8')
                        site = getSite(s,st)
                        t = translations[s][projects[fam]['template']]
                        c = translations[s]['Bot: Updating statistics']
                        print t
                        page = Page(s,t.decode('utf-8'))
                        page.put(content,comment=c.decode('utf-8'))