#!/usr/bin/perl
#-----------------------------------------------------------------------------
#
#  Tirex Tile Rendering System
#
#  tirex-tiledir-check
#
#-----------------------------------------------------------------------------
#  See end of this file for documentation.
#-----------------------------------------------------------------------------
#
#  Copyright (C) 2010  Frederik Ramm <frederik.ramm@geofabrik.de> and
#                      Jochen Topf <jochen.topf@geofabrik.de>
#  
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#  
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with this program; If not, see <http://www.gnu.org/licenses/>.
#
#-----------------------------------------------------------------------------

use strict;
use warnings;

use File::Find;
use File::stat;
use Getopt::Long qw( :config gnu_getopt );
use IO::File;
use IO::Handle;
use IO::Seekable;
use JSON;
use List::Util qw();
use Pod::Usage qw();

use Tirex;
use Tirex::Renderer;
use Tirex::Map;

#-----------------------------------------------------------------------------

my $EMPTY_TILE_SIZE = 7124;

my %opts = ( list => '', stats => '' );
GetOptions( \%opts, 'help|h', 'config|c=s', 'list|l=s', 'stats|s=s', 'minz|z=i', 'maxz|Z=i' ) or exit(2);

if ($opts{'help'})
{
    Pod::Usage::pod2usage(
        -verbose => 1,
        -msg     => "tirex-tiledir-check - check tile dir\n",
        -exitval => 0
    );
}

die("--list and --stats can't both be -\n") if ($opts{'list'} eq '-' && $opts{'stats'} eq '-');

my $fh_list;
my $fh_stats;

if ($opts{'list'} eq '-')
{
    $fh_list = IO::Handle->new();
    $fh_list->fdopen(fileno(STDOUT), 'w');
}
elsif ($opts{'list'})
{
    $fh_list = IO::File->new($opts{'list'}, 'w') or die("Can't open list file '$opts{'list'}': $!\n");
}

if ($opts{'stats'} eq '-')
{
    $fh_stats = IO::Handle->new();
    $fh_stats->fdopen(fileno(STDOUT), 'w');
}
elsif ($opts{'stats'})
{
    $fh_stats = IO::File->new($opts{'stats'}, 'a') or die("Can't open stats file '$opts{'stats'}': $!\n");
}

die("missing map parameter\n") unless (defined $ARGV[0]);
my $mapname = shift;

my $config_dir = $opts{'config'} || $Tirex::TIREX_CONFIGDIR;
my $config_file = $config_dir . '/' . $Tirex::TIREX_CONFIGFILENAME;
Tirex::Config::init($config_file);

Tirex::Renderer->read_config_dir($config_dir);

my $map = Tirex::Map->get($mapname);
die("unknown map: $mapname\n") unless (defined $map);

#-----------------------------------------------------------------------------

my $REGEX_DIR  = qr{^(/[0-9]+){0,5}$};
my $REGEX_FILE = qr{^(/[0-9]+){0,5}/[0-9]+\.meta$};
my $REGEX_TMP  = qr{^(/[0-9]+){0,5}/[0-9]+\.meta\.[0-9]+\.tmp$};

my $BLOCKSIZE = 512;

my $mtdir = $map->get_tiledir();
chop $mtdir if ($mtdir =~ qr{/$});

my %stats;
my $exitcode = 0;
my $list = '';

my $minz = $opts{'minz'};
$minz = 0 unless defined($minz);
my $maxz = $opts{'maxz'};
$maxz = 19 unless defined($maxz);

for (my $i=$minz; $i<=$maxz; $i++)
{
    next unless (-d "$mtdir/$i");
    find({ wanted => \&process, follow_fast => 1, no_chdir => 1 }, "$mtdir/$i");
    if ($fh_list)
    {
        $fh_list->print($list);
    }
    $list = '';
}

if ($fh_stats)
{
    my $json = JSON::to_json(\%stats, { pretty => 1 });
    unless ($opts{'stats'} eq '-')
    {
        $fh_stats->truncate(0);
        $fh_stats->seek(0, SEEK_SET);
    }
    $fh_stats->print("$json\n");
}

exit($exitcode);

#-----------------------------------------------------------------------------

sub error
{
    my $errmsg = shift;

    print STDERR "$errmsg\n";
    $exitcode = 1;
}

sub process
{
    (my $path = $_) =~ s/^$mtdir//;

    # directories
    if (-d $_)
    {
        return if ($path eq '');

        if ($path !~ $REGEX_DIR)
        {
            $File::Find::prune = 1;
            error("Unknown directory format: $path");
        }

        return;
    }

    my $stat = stat($_);

    # is a tmp file
    if ($path =~ $REGEX_TMP)
    {
        #  older than 1 minute
        error("Old tmp file: $path") if ($stat->mtime() < time() - 60);
        return;
    }

    # not a metatile file
    if ($path !~ $REGEX_FILE)
    {
        error("Unknown file format: $path");
        return;
    }

    my $metatile = Tirex::Metatile->new_from_filename_and_map($path, $mapname);

    if (!$stat)
    {
        error("Can't stat file: $path");
        return;
    }

    my $age    = time() - $stat->mtime();
    my $size   = $stat->size();
    my $blocks = $stat->blocks();

    $list .= join(',', $age, $size, $blocks, $metatile->to_s()) . "\n" if ($fh_list);

    return unless ($fh_stats);

    if (! defined $stats{$metatile->get_map()}->[$metatile->get_z()])
    {
        $stats{$metatile->get_map()}->[$metatile->get_z()] = {
            'minage'     => 999_999_999,
            'maxage'     => 0,
            'sumage'     => 0,
            'minsize'    => 999_999_999,
            'maxsize'    => 0,
            'sumsize'    => 0,
            'minblocks'  => 999_999_999,
            'maxblocks'  => 0,
            'sumblocks'  => 0,
            'count'      => 0,
            'countempty' => 0,
        };
    }

    my $s = $stats{$metatile->get_map()}->[$metatile->get_z()];

    $s->{'minage'}    = 0 + List::Util::min($s->{'minage'}   , $age);
    $s->{'maxage'}    = 0 + List::Util::max($s->{'maxage'}   , $age);
    $s->{'minsize'}   = 0 + List::Util::min($s->{'minsize'}  , $size);
    $s->{'maxsize'}   = 0 + List::Util::max($s->{'maxsize'}  , $size);
    $s->{'minblocks'} = 0 + List::Util::min($s->{'minblocks'}, $blocks);
    $s->{'maxblocks'} = 0 + List::Util::max($s->{'maxblocks'}, $blocks);
    $s->{'sumage'}    += $age;
    $s->{'sumsize'}   += $size;
    $s->{'sumblocks'} += $blocks;
    $s->{'count'}++;
    $s->{'countempty'}++ if ($size == $EMPTY_TILE_SIZE);
}

__END__

=head1 NAME

tirex-tiledir-check - check tile dir

=head1 SYNOPSIS

tirex-tiledir-check [OPTIONS] MAP

=head1 OPTIONS

=over 8

=item B<-h>, B<--help>

Display help message.

=item B<-c>, B<--config=DIR>

Use the config directory DIR instead of /etc/tirex.

=item B<-l>, B<--list=FILE>

Write list of tiles to FILE. See below for format.

=item B<-s>, B<--stats=FILE>

Write stats about tiles to FILE. See below for format.

=item B<-z>, B<--minz=i>

Start processing at min. zoom level i (default=0)

=item B<-Z>, B<--maxz=i>

Stop processing at max. zoom level i (default=19)

=back

=head1 DESCRIPTION

Walks recursively through the tiles directory for the given MAP and checks for
wrong directory or file names or files that can't be accessed. If problems are
found, messages are written to STDERR.

When the --list and/or --stats options are given, it outputs information about
each tile and/or generates statistics, respectively. The list and stats are
written to the filenames given, if you give '-' as the filename STDOUT is used.
You can only use STDOUT for either the list file or the stats file.

Caution: This command will go through all metatiles on your disk and stat each
file. On a lightly loaded machine with lots of RAM this can go pretty quick,
but under high IO loads it might take a long time. Take this into account if
you want to run it regularly from cron or similar.

=head1 LIST FILE FORMAT

The list file is in CSV format with one line per metatile. The
fields are:

=over 8

=item age

Age of metatile in seconds.

=item size

Size in bytes of the metatile.

=item blocks

Number of blocks used for this metatile.

=item metatile

Description of metatile. Format: map=foo x=8 y=0 z=10

=back

This data can be read with other programs to create statistics etc.

=head1 STATS FILE FORMAT

The stats file is in JSON format. It contains a hash with the names
of all maps as keys and a list of zoom levels as their values. For
each zoom level there is a nested hash containing the statistics:

=over 8

=item count - Number of tiles

=item maxage - Maximum age in seconds

=item maxblocks - Maximum number of blocks

=item maxsize - Maximum file size in bytes

=item minage - Minimum age in seconds

=item minblocks - Minimum number of blocks

=item minsize - Minimum file size in bytes

=item sumage - Sum of ages in seconds

=item sumblocks - Sum of number of blocks

=item sumsize - Sum of file size in bytes

=back

The sumage, sumblocks, and sumsize values can be divided by count to get
the average.

This file can be displayed on a human readable format with the Program
tirex-tiledir-stat. It is also read by several Munin plugins.

=head1 FILES

=over 8

=item F</etc/tirex/tirex.conf>

The configuration file.

=back

=head1 DIAGNOSTICS

Returns 0 if no errors were found or 1 if there were errors. If there were
errors parsing the command line 2 is returned.

=head1 SEE ALSO

L<http://wiki.openstreetmap.org/wiki/Tirex>

=head1 AUTHORS

Frederik Ramm <frederik.ramm@geofabrik.de>, Jochen Topf
<jochen.topf@geofabrik.de> and possibly others.

=cut


#-- THE END ------------------------------------------------------------------
