User:LA2/Extraktor

#!/usr/bin/perl -w 
 
use utf8;
# use encoding 'utf8';
use locale;
use POSIX;
use strict;
# setlocale(LC_CTYPE, "UTF-8");
 
# Treat <ref> tags as a kind of template call
sub reftag ()
{
    my ($pagename, $seqno, $name, $value) = @_;
    if (defined($name)) {
        print "<ref>|$pagename|$seqno|name|$name\n";
    }
    if (defined($value)) {
        # escape any remaining vertical bars
        $value =~ s/(http:[^ ]*)\|/$1%7c/g;
        $value =~ s/\|/\$!/g;
        print "<ref>|$pagename|$seqno|1|$value\n";
    }
    return "\@(<ref>:$seqno)";
}
 
 # Parse arguments of one template call
sub template ()
{
    my ($tempname, $pagename, $seqno, $args) = @_;
 
    # {{foo bar}} calls Template:Foo_bar with upper case F.
    # Might not work for non-ASCII characters.
    $tempname = ucfirst($tempname);
    $tempname =~ s/ /_/g;
 
    if  (defined ($args)) {
        my ($field, $value);
        my $argc = 0;
        $args =~ s/^\|\s*(.*?)\s*$/$1/;
        foreach my $arg (split(/\s*\|\s*/, $args)) {
            $argc++;
            if ($arg =~ /^([^=]*?)\s*=\s*(.*)$/) {
                $field = $1;
                $value = $2;
            } else {
                $field = $argc;
                $value = $arg;
            }
            print "$tempname|$pagename|$seqno|$field|$value\n";
        }
    } else {
        # Template call without parameters. Only print three fields
        print "$tempname|$pagename|$seqno\n";
    }
    return "\@($tempname:$seqno)";
}
 
# This is like sub template() but for magic keywords RFC and ISBN that
# are followed by just one argument.
sub magicword ()
{
    my ($tempname, $pagename, $seqno, $arg) = @_;
    print "$tempname|$pagename|$seqno|1|$arg\n";
    return "$tempname $arg";
}
 
# Parse Wikipedia pages-articles XML dump
my ($title, $text, $append) = ("", "", 0);
while (<>) {
    if (/\<title\>(.*)\<\/title\>/) {
        $title = $1;
        $text = "";
        next;
    }
    $append = 1 if /\<text/;
    $text .= $_ if $append;
    if (/\<\/text\>/) {
        my $i;
        $append = 0;
        $text =~ s/\n/ /g;
 
        $text =~ s/.*\<text[^\>]*\>(.*)\<\/text\>.*/$1/;
 
        # Perform various substitutions to get rid of troublesome
        # wiki markup.  In its place, leave $something
 
        # silently drop HTML comments
        $text =~ s/&amp;lt;!--.*?--&amp;gt;//g;
 
        # ignore nowiki, non-greedy match, leave $nowiki
        $text =~ s/&amp;lt;nowiki&amp;gt;.*?&amp;lt;\/nowiki&amp;gt;/\$nowiki/g;
 
        # ignore math, non-greedy match, leave $math
        $text =~ s/&amp;lt;math&amp;gt;.*?&amp;lt;\/math&amp;gt;/\$math/g;
 
        # wiki link with alternative text, leave $!
        # multiple passes handle image thumbnails
        for ($i = 0; $i < 5; $i++) {
            $text =~ s/(\[\[[^\]\|{}]*)\|([^\]{}]*\]\])/$1\$!$2/g;
        }
 
        # These are not real template calls, leave $pagename
        $text =~ s/{{(CURRENT(DAY|DOW|MONTH|TIME(STAMP)?|VERSION|WEEK|YEAR)(ABBREV|NAME(GEN)?)?|(ARTICLE|NAME|SUBJECT|TALK)SPACE|NUMBEROF(ADMINS|ARTICLES|FILES|PAGES|USERS)(:R)?|(ARTICLE|BASE|FULL|SUB|SUBJECT|TALK)?PAGENAMEE?|REVISIONID|SCRIPTPATH|SERVER(NAME)?|SITENAME)}}/\$$1/g;
 
        # template parameter value with default, leave $!
        $text =~ s/{{{([^\|{}]*)\|([^{}]*)}}}/\$($1\$!$2)/g;
 
        # template parameter values, leave $parameter
        $text =~ s/{{{([^{}]*)}}}/\$($1)/g;
 
        # template bang escape, leave $!
        $text =~ s/{{!}}/\$!/g;
 
        my $seqno = 1;
        $text =~ s/(ISBN) +([0-9][- 0-9Xx]+[0-9Xx)/&magicword($1,$title,$seqno++,$2)/eg;
        $text =~ s/(RFC) +([0-9]+)/&magicword($1,$title,$seqno++,$2)/eg;
        # multiple passes handle nested template calls
        for ($i = 0; $i < 5; $i++) {
            # pretend that <ref> tags are a kind of template call
            $text =~ s/&amp;lt;ref(\s+name\s*=\s*(&amp;quot;)?([^&amp;\s]*[^&amp;\s\/])(&amp;quot;)?)?\s*(\/|&amp;gt;([^{}]*?)&amp;lt;\/ref\s*)&amp;gt;/&amp;reftag($title,$seqno++,$3,$6)/eg;
            $text =~ s/{{\s*([^\|{}]*?)\s*(\|[^{}]*)?}}/&amp;template($1,$title,$seqno++,$2)/eg;
        }
 
        # Debugging
        # print "$title<>$text\n";
    }
}