Language tools/checkutftable

Note that program is under GNU GPL, not under GNU FDL. If you contribute to this program, please, add your name inside of copyright notice.

I made this module for checking is the text on Serbian Wikipedia in Cyrillic or in Latin script. However, I realized that it can be useful for other languages, too. Characters which do not belong strictly to any script are described as 'not important'. If character is not recognized by unicodedata module, it is described as 'not known'.

#!/usr/bin/python
#
# Copyright (C) 2006 Milos Rancic
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# 
# Text of GNU GPL license can be found at http://www.gnu.org/licenses/gnu.html

import sys
import unicodedata

# engine
def check(f):
        text = file(f).read().decode('utf-8');
        numchar = {}
        numchar['all'] = 0
        numchar['ni'] = 0 #not important
        numchar['nnnn'] = 0 #not known
        
        numchar['ARAB'] = 0 #ARABIC
        numchar['ARME'] = 0 #ARMENIAN
        numchar['BENG'] = 0 #BENGALI
        numchar['BOPO'] = 0 #BOPOMOFO
        numchar['BRAI'] = 0 #BRAILLE PATTERN
        numchar['BUHI'] = 0 #BUHID
        numchar['CANA'] = 0 #CANADIAN
        numchar['CHER'] = 0 #CHEROKEE
        numchar['CJK '] = 0 #CJK
        numchar['COMB'] = 0 #COMBINING
        numchar['COPT'] = 0 #COPTIC
        numchar['CYRI'] = 0 #CYRILLIC
        numchar['DESE'] = 0 #DESERET
        numchar['DEVA'] = 0 #DEVANGARI
        numchar['ETHI'] = 0 #ETHIOPIC
        numchar['EXTE'] = 0 #EXTENDED ARABIC
        numchar['GEOR'] = 0 #GEORGIAN
        numchar['GOTH'] = 0 #GOTHIC
        numchar['GREE'] = 0 #GREEK
        numchar['GUJA'] = 0 #GUJARATI
        numchar['GURM'] = 0 #GURMUKHI
        numchar['HANG'] = 0 #HANGUL
        numchar['HANU'] = 0 #HANUNOO
        numchar['HEBR'] = 0 #HEBREW
        numchar['HIRA'] = 0 #HIRAGANA
        numchar['IDEO'] = 0 #IDEOGRAPHIC
        numchar['KANG'] = 0 #KANGXI RADICAL
        numchar['KANN'] = 0 #KANNADA
        numchar['KATA'] = 0 #KATAKANA
        numchar['KHME'] = 0 #KHMER
        numchar['LAO '] = 0 #LAO 
        numchar['LATI'] = 0 #LATIN
        numchar['MALA'] = 0 #MALAYALAM
        numchar['MODI'] = 0 #MODIFIER LETTER
        numchar['MONG'] = 0 #MONGOLIAN
        numchar['MYAN'] = 0 #MYANMAR
        numchar['OGHA'] = 0 #OGHAM
        numchar['OLD '] = 0 #OLD ITALIC
        numchar['ORIY'] = 0 #ORIYA
        numchar['PHIL'] = 0 #PHILIPPINE
        numchar['RUNI'] = 0 #RUNIC
        numchar['SINH'] = 0 #SINHALA
        numchar['SYRI'] = 0 #SYRIAC
        numchar['TAGA'] = 0 #TAGALOG
        numchar['TAGB'] = 0 #TAGBANWA
        numchar['TAMI'] = 0 #TAMIL
        numchar['TELU'] = 0 #TELUGU
        numchar['THAA'] = 0 #THAANA
        numchar['THAI'] = 0 #THAI
        numchar['TIBE'] = 0 #TIBETAN
        numchar['YI R'] = 0 #YI RADICAL
        numchar['YI S'] = 0 #YI SYLLABLE

        for t in range(0,len(text)):
                character = unicodedata.name(text[t],"nnnn")
                if len(character) < 4:
                        character += ' '
                char = character[0:4]
                if char not in numchar:
                        char = 'ni'
                numchar[char] += 1
                numchar['all'] += 1
        return numchar

# the list of existing character groups
def describe(d):
        perc = {}
        out = {}
        for n in d:
                if (d[n] > 0) and (n != 'all'):
                        perc[n] = float(d[n])/float(d['all'])
                        out[n] = [ n, perc[n] ]
        return out

# what group of characters has absolute majority, what relative
def decide(d):
        izbor = ''
        perc = {}
        ctrl = 0
        for n in d:
                if (d[n] > 0) and (n != 'all'):
                        perc[n] = float(d[n])/float(d['all'])
                        if perc[n] > ctrl:
                                ctrl = perc[n]
                                czbor = n
                        if perc[n] > 0.5:
                                izbor = n
        if izbor == '':
                out = [ 'r', czbor, ctrl ]
        else:
                out = [ 'a', izbor, perc[izbor] ]
        return out

# for Serbian (Wikipedia): what is the script of the text
def analyzecyr(d):
        izbor = ''
        perc = {}
        ctrl = 0
        if 'CYRI' in d and 'LATI' in d:
                d['cyrlat'] = d['CYRI'] + d['LATI']
                perc['CYRI'] = float(d['CYRI'])/float(d['cyrlat'])
                perc['LATI'] = float(d['LATI'])/float(d['cyrlat'])
                if perc['CYRI'] > 0.5:
                        ra = 'a'
                        izbor = 'CYRI'
                elif perc['CYRI'] > 0.3:
                        ra = 'r'
                        izbor = 'CYRI'
                elif perc['LATI'] > 0.5:
                        ra = 'a'
                        izbor = 'LATI'
                elif perc['LATI'] > 0.3:
                        ra = 'r'
                        izbor = 'LATI'
                else:
                        ra = 'n'
                        izbor = 'n'
                        perc['n'] = 0
        out = [ ra, izbor, perc[izbor] ]
        return out      

# if the program is started from the command line
try:
        fl = sys.argv[1]
        dictionary = check(fl)
        print describe(dictionary)
        print decide(dictionary)
        print analyzecyr(dictionary)
except IndexError:
        pass