#!/usr/bin/perl
use strict;
use warnings;
use Statistics::ChisqIndep;
use DB_File;
use Carp;

=end
use 5.34.0;
use feature 'try';
local $SIG{__WARN__} = sub {
die;
}; # turns warnings into exceptions
=cut

my ($filename, $howmanywords) = @ARGV;
$howmanywords ||= 5;

open FILE, "<", $filename or croak $!;

my (%words, $total);

while (<FILE>) {
	while ( /(\w[\w'-]*)/g ) {
		$total++;
		$words{lc $1}++ unless substr($1, 0, 1) =~ /[A-Z]/;
	}
}

close FILE;

tie my %wordfreqpos, "DB_File", "wf4", O_RDONLY;

my %abouttags;
my $chi = new Statistics::ChisqIndep;

while(my ($word, $observedfreq) = each(%words)) {
	my ($freq, $pos);
=end
	try {
		($freq, $pos) = split ':', $wordfreqpos{$word};
	}
	catch ($e) {
		print "$word\n";
	}
=cut
	if (exists $wordfreqpos{$word}) {
		($freq, $pos) = split $;, $wordfreqpos{$word};
		print "$word $freq, $pos\n";
	} else {
		print "missing $word\n";
	}
	$freq ||= 1000; # put in a thousand you idiot!
	$pos ||= 'nn';
	my $expected = $total*$freq/100104253;
#	print "$word $expected\n" if $word eq "good";
	my @obs = ([$observedfreq, $total - $observedfreq], [$expected, $total - $expected]);
	$chi->load_data(\@obs);
	if ($pos =~ /av|vv|nn/ && $pos !~ /unc/) {
		$abouttags{$word} = $chi->{p_value}
	}
}

my @list = sort { $abouttags{$a} <=> $abouttags{$b} } keys %abouttags;

for (0..$howmanywords - 1) {
#	my $word = pop @list;
#	print "$word\n";
	print $list[$_], "\n";
}

for (0..$howmanywords - 1) {
	my ($freq, $pos) = split $;, $wordfreqpos{$list[$_]};
	print "$list[$_]:$freq:$pos\n" 
}

my @r = reverse @list;

for (0..$howmanywords - 1) {
	print $r[$_], "\n";
}

untie %wordfreqpos;
