#!/usr/bin/perl -w
# NAME: osm-load.pl
# AIM: Load an XML OSM file
use strict;
use warnings;
use File::Basename;  # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] )
use Cwd;
my $perl_dir = 'C:\GTools\perl';
unshift(@INC, $perl_dir);
#require 'logfile.pl' or die "Unable to load logfile.pl ...\n";
require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl'! Check location and \@INC content.\n";
require 'fg_wsg84.pl' or die "Unable to load fg_wsg84.pl ...\n";
require "Bucket2.pm" or die "Unable to load Bucket2.pm ...\n";
# log file stuff
our ($LF);
my $pgmname = $0;
if ($pgmname =~ /(\\|\/)/) {
    my @tmpsp = split(/(\\|\/)/,$pgmname);
    $pgmname = $tmpsp[-1];
}
my $outfile = $perl_dir."\\temp.$pgmname.txt";
open_log($outfile);

# user variables
my $load_log = 1;
my $in_file = '';
my $SG_METER_TO_NM = 0.0005399568034557235;
my $temp_poly = $perl_dir."\\temppoly.";
my $poly_count = 0;

# standard fields to poulate in Tags
my @standardFields = ('highway','junction','cycleway','tracktype','waterway','railway',
                     'aeroway','aerialway','power','man_made','leisure',
                     'amenity','shop','tourism','historic','landuse',
                     'military','natural','route','boundary','sport',
                     'abutters','fenced','lit','width','lanes',
                     'bridge','tunnel','cutting','embankment','layer',
                     'surface','name','int_name','nat_name','reg_name',
                     'loc_name','old_name','ref','int_ref','nat_ref',
                     'reg_ref','loc_ref','old_ref','ncn_ref','place',
                     'place_name','place_numbers','postal_code','is_in','note','class');
sub in_standard($) {
    my $tag = shift;
    my ($tt);
    foreach $tt (@standardFields) {
        return 1 if ($tt eq $tag);
    }
    return 0;
}
#Tags with these keys will not be loaded, features with *only* these keys will not be loaded
my @ignoreFields = ('created_by','source','converted_by');
#this flag controls whether features with only non standard tags are loaded.
my $loadNonstandardTags = 0;

my $debug_on = 1;
my $def_file = 'C:\Documents and Settings\Geoff McLane\My Documents\FG\OSM\YGIL-map.osm';

### program variables
my @warnings = ();
my $cwd = cwd();
my $os = $^O;
my $g_minlat = 200;
my $g_maxlat = -200;
my $g_minlon = 200;
my $g_maxlon = -200;
my ($g_center_lat,$g_center_lon);
my ($g_lat_span,$g_lon_span);
my ($g_width,$g_height);

sub show_warnings($) {
    my ($val) = @_;
    if (@warnings) {
        prt( "\nGot ".scalar @warnings." WARNINGS...\n" );
        foreach my $itm (@warnings) {
           prt("$itm\n");
        }
        prt("\n");
    } else {
       # prt( "\nNo warnings issued.\n\n" );
    }
}

sub pgm_exit($$) {
    my ($val,$msg) = @_;
    if (length($msg)) {
        $msg .= "\n" if (!($msg =~ /\n$/));
        prt($msg);
    }
    show_warnings($val);
    close_log($outfile,$load_log);
    exit($val);
}


sub prtw($) {
   my ($tx) = shift;
   $tx =~ s/\n$//;
   prt("$tx\n");
   push(@warnings,$tx);
}

#---------------------------------------------------------------------------
#Gets the XML element name from the string passed in
#an end of element tag is /element
#---------------------------------------------------------------------------
sub getElement($) {
    my $line = shift;
    my $el = '';
    my $s = index($line,'<');
    if ($s > 0) {
        my $e = index($line,' ',$s);
        if ($e > 0) {
            $el = substr($line,$s+1,$e-$s-1);
        } else {
            $e = index($line,'>',$s);
            if ($e > 0) {
                $el = substr($line,$s+1,$e-$s-1);
                $el =~ s/\/$//;
            } else {
                prtw("WARNING: NO ELEMENT [$line]?\n");
            }
        }
    }
    return $el;
}

#---------------------------------------------------------------------------
#Gets the value of the named attribute from the string
#---------------------------------------------------------------------------
sub getAttributeValue($$) {
    my ($name,$line) = @_;
    my $attr = '';
    my $sa = index($line,' '.$name.'="');
    if ($sa > 0) {
        #prt("Begin = $sa, ");
        $sa += length($name) + 3; # past the first '"'
        my $ea = index($line,'"',$sa);
        if ($ea > 0) {
            #prt("Begin = $sa, End = $ea\n");
            $attr = substr($line,$sa,$ea - $sa);
        }
    }
    return $attr;
}
#---------------------------------------------------------------------------    

#---------------------------------------------------------------------------
#Extract Node attribute details from a line of xml text
#---------------------------------------------------------------------------
sub returnNode($) {
    my $line = shift;
    my $nid = getAttributeValue('id',$line);
    my $nx = getAttributeValue('lon',$line);
    my $ny = getAttributeValue('lat',$line);
    return($nid,$nx,$ny);
}
#---------------------------------------------------------------------------
#---------------------------------------------------------------------------
#extract segment attributes from a line of xml text
#---------------------------------------------------------------------------
sub returnSegment($) {
    my $line = shift;
    my $sid = getAttributeValue('id',$line);
    my $sn = getAttributeValue('from',$line);
    my $en = getAttributeValue('to',$line);
    return($sid,$sn,$en);
}
#---------------------------------------------------------------------------
    
#get the id attribute from a line of xml text
#used for ways and its segs, as id is only attribute needed
sub returnID($) {
    my $line = shift;
    return getAttributeValue('id',$line);
}

sub returnTags($) {
    my $line = shift;
    my $k = getAttributeValue('k',$line); # .lstrip().encode("Latin-1","replace")[:29]
    my $v = getAttributeValue('v',$line); # .encode("Latin-1","replace")[:254]
    return ($k,$v);
}

#---------------------------------------------------------------------------
sub in_world_range($$) {
    my ($lat,$lon) = @_;
    return 0 if ($lat < -90);
    return 0 if ($lat > 90);
    return 0 if ($lon < -180);
    return 0 if ($lon > 180);
    return 1;
}

#node=('ID','x','y')
#segment=('id','start','end')
#way=('id','seg1 seg2')
#tag=('key','value')

sub process_in_file($) {
    my ($inf) = @_;
    if (! open INF, "<$inf") {
        pgm_exit(1,"ERROR: Unable to open file [$inf]\n"); 
    }
    my @lines = <INF>;
    close INF;
    my $lncnt = scalar @lines;
    prt("Processing $lncnt lines, from [$inf]...\n");
    my ($line,$inc,$lnn);
    my ($element,$ftype,$id,$lon,$lat,$v,$k,$val);
    my ($minlat,$minlon,$maxlat,$maxlon,$hadbounds,$msg);
    my %nodes = ();
    my %node_tags = ();
    my %way_tags = ();
    my %way_nd = ();
    my @NODES = ();
    my @ftags = ();
    my @unbuiltways = ();
    my @ways = ();
    my @nd_array = ();
    $lnn = 0;
    $ftype = -1;
    $hadbounds = 0;
    my $hasvalidtags = 0;
    my $nodecount = 0;
    my $waycount = 0;
    my $way = 0;
    my $waytagcount = 0;
    my $nodetagcount = 0;
    my $taggednodecount = 0;
    foreach $line (@lines) {
        chomp $line;
        $lnn++;
        $element = getElement($line);
        next if (length($element) == 0);
        if ($element eq 'node') {
            $ftype = -1;
            @ftags = ();
            ($id,$lon,$lat) = returnNode($line);
            #if float(node[1])>=-180 and float(node[1])<=180 and float(node[2])>=-90 and float(node[2])<=90:
            #if ( ($lon >= -180) && ($lon <= 180) && ($lat >= -90) && ($lat <= 90) && ($id > 0)) {
            if (in_world_range($lat,$lon)) {
                $ftype = 0; # set a VALID 'node'
                #nodefile.write(str(node[0])+':'+str(node[1])+':'+str(node[2])+'\n')
                push(@NODES, [$lon,$lat]);
                #  <bounds minlat="-31.7130540" minlon="148.6136050" maxlat="-31.6947200" maxlon="148.6567780"/>
                if (defined $nodes{$id}) {
                    prtw("WARNING:$lnn: Node ID [$id] REPEATED\n");
                } else {
                    $nodes{$id} = [$lon,$lat];
                }
                # oops, found items CAN be out of this range - so no range check
                #if ($hadbounds && (($lat < $minlat)||($lat > $maxlat)||($lon < $minlon)||($lon > $maxlon))) {
                #    $msg = "lat,lon $lat,$lon OUT OF BOUNDS ";
                #    prtw("WARNING:$lnn: $msg\n");
                #}
                # but keep the NEW ranges
                $g_minlat = $lat if ($lat < $g_minlat);
                $g_maxlat = $lat if ($lat > $g_maxlat);
                $g_minlon = $lon if ($lon < $g_minlon);
                $g_maxlon = $lon if ($lon > $g_maxlon);
                $nodecount++;
            } else {
                prtw("WARNING:$lnn: Line $id,$lon,$lat [".substr($line,0,70)."...] FAILED!\n");
            }
        } elsif ($element eq 'way') {
            #  <way id="37623654" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812676" timestamp="2009-07-13T00:57:47Z">
            @ftags = ();
            $ftype = 2;
            $waycount++;
            $way = returnID($line); # set the way
            if (length($way)) {
                push(@ways,$way);
            } else {
                prtw("WARNING:$lnn: way element without ID [$way] line [$line]\n");
            }
            # unbuiltways.write('\n'+str(way[0])+'#')
        } elsif ($element eq 'nd') {
            $val = getAttributeValue('ref',$line);
            if (length($val)) {
                push(@nd_array,$val);
            } else {
                prtw("WARNING:$lnn: nd element without REF [$val] line [$line]\n");
            }
            # unbuiltways.write(str(getAttributeValue('ref',line))+':')
        } elsif ($element eq 'tag' ) {
            ($v,$k) = returnTags($line);
            # prt("tag element: v=$v k=$k ftype=$ftype\n");
            if ($ftype == 0) {
                # tagged node
                #ignore less useful tags, and if not a standard tag
                #remove tags with blank values too. lots of wierd keys have blank values
                if ((length($v) == 0)||(length($k) == 0)) {
                    prtw("WARNING:$lnn:1: ftype=$ftype v=$v, k=$k\n");
                }else {
                    #if tag[0] in standardFields and tag[1] !='':
                    if (in_standard($v)) {
                        #ftags.append((tag[0],tag[1]))
                        push(@ftags, [$v,$k]);
                        $hasvalidtags = 1;
                        $nodetagcount++;
                    } elsif ($loadNonstandardTags) {
    #                    if tag[0] not in ignoreFields and tag[0] not in standardFields and tag[1] !='':
    #                        hasvalidtags=True
    #                        row=othernodetagcursor.newrow()
    #                        row.Node_ID=long(node[0])
    #                        row.Tag_Name=str(tag[0])
    #                        row.Tag_Value=str(tag[1])
    #                        othernodetagcursor.insertrow(row)
    #                        nodetagcount+=1
                    }
                }
            } elsif ($ftype == 2) {
                #way tags, loading all these except ignorefields and blank valued ones.
                if ((length($v) == 0)||(length($k) == 0)) {
                    prtw("WARNING:$lnn:2: ftype=$ftype v=$v, k=$k\n");
                } else {
                    # if tag[0] in standardFields and tag[1] !='':
                    if ( in_standard($v) ) {
                        push(@ftags, [$v,$k]);
                        #ftags.append((tag[0],tag[1]))
                        $hasvalidtags = 1;
                        $waytagcount++;
                    } elsif ($loadNonstandardTags) {
                         #if tag[0] not in ignoreFields and tag[0] not in standardFields and tag[1] !='':
                         #    row=otherwaytagcursor.newrow()
                         #    row.Way_ID=long(way[0])
                         #    row.Tag_Name=str(tag[0])
                         #    row.Tag_Value=str(tag[1])
                         #    otherwaytagcursor.insertrow(row)
                         #    waytagcount+=1
                    }
                }
            } else {
                prtw("WARNING:$lnn:3: ftype=$ftype v=$v, k=$k\n");
            }
        } elsif (($element eq '/node') && $hasvalidtags && ($ftype == 0)) {
            # <node id="441007981" lat="-31.7088281" lon="148.6651552" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812676" timestamp="2009-07-13T00:57:45Z"/>
            # <node id="441013712" lat="-31.7020985" lon="148.6473310" user="nm7s9" uid="12434" visible="true" version="1" changeset="1812736" timestamp="2009-07-13T01:35:23Z">
            # <tag k="railway" v="level_crossing"/>
            # </node>
            if (defined $node_tags{$id}) {
                prtw("WARNING:$lnn: Duplicated ID in node_tags [$id]\n");
            }
            my @a1 = @ftags;
            $node_tags{$id} = \@a1;
            @ftags = ();
            #done with node lets load its shape
            # frow = nodecursor.newrow()
            #nodepnt.x=float(node[1])
            #nodepnt.y=float(node[2])
            # frow.setValue("Node_ID",node[0].encode("Latin-1","replace"))
            # for f in standardFields:
            #   frow.setValue(f,'')
            # for sTag in ftags:
#                frow.setValue(sTag[0],str(sTag[1]))
#            frow.SetValue('shape', nodepnt) #Load the shape
#            nodecursor.insertrow(frow)
            $taggednodecount++;
            $hasvalidtags = 0;
            #$ftype = -1;
        } elsif (($element eq '/way') && $hasvalidtags) {
            #  <way id="30215921" user="VK1RE" uid="69499" visible="true" version="2" changeset="808028" timestamp="2009-01-18T11:05:50Z">
            #   <nd ref="332988044"/>
            #   <nd ref="332988046"/>
            #  <tag k="created_by" v="Potlatch 0.10f"/>
            #  <tag k="highway" v="residential"/>
            # </way>
            #done with way lets load attributes, shape comes later
            my @a2 = @ftags;
            if (defined $way_tags{$way}) {
                prtw("WARNING:$lnn: Duplicated ID in way_tags [$way]\n");
            }
            $way_tags{$way} = \@a2;
            #  trow=waytagcursor.newrow()
            #    trow.SetValue("Way_ID",long(way[0]))
            #     for f in standardFields:
            #         trow.setValue(f,'')
            #     for sTag in ftags:
            #         trow.setValue(sTag[0],str(sTag[1]))
            #     waytagcursor.insertrow(trow)
            if (defined $way_nd{$way}) {
                prtw("WARNING:$lnn: Duplicated ID in way_nd [$way]\n");
            }
            my @a3 = @nd_array;
            $way_nd{$way} = \@a3;
            @nd_array = ();
            @ftags = ();
            $hasvalidtags = 0;
            #$ftype = -1;
        } elsif ($element eq 'member') {
            # 
        } elsif ($element eq 'bounds') {
            #  <bounds minlat="-31.7130540" minlon="148.6136050" maxlat="-31.6947200" maxlon="148.6567780"/>
            $hadbounds = 1;
            $minlat = getAttributeValue('minlat',$line);
            $minlon = getAttributeValue('minlon',$line);
            $maxlat = getAttributeValue('maxlat',$line);
            $maxlon = getAttributeValue('maxlon',$line);
            prt("bounds $minlat,$minlon $maxlat,$maxlon\n");
        } elsif ($element eq 'relation') {
            #
        } elsif ($element eq '/relation') {
            #
        } else {
            prtw("WARNING:$lnn: Element [$element] NOT dealt with\n");
        }
    }
    prt( "Verticies=$nodecount, ntc=$nodetagcount, way=$waycount, waytc=$waytagcount, tnc=$taggednodecount\n");
    prt("tag  bounds $minlat,$minlon $maxlat,$maxlon\n");
    $g_center_lat = ($g_minlat + $g_maxlat) / 2;
    $g_center_lon = ($g_minlon + $g_maxlon) / 2;
    $g_lat_span = abs($g_maxlat - $g_minlat);
    $g_lon_span = abs($g_maxlon - $g_minlon);
    $g_width = int($g_lon_span * 1000)+1;
    $g_height = int($g_lat_span * 1000)+1;
    prt("glob bounds $g_minlat,$g_minlon $g_maxlat,$g_maxlon\n");
    prt("Center $g_center_lat,$g_center_lon, span $g_lat_span,$g_lon_span h=$g_height w=$g_width\n");
    my %hash = ();
    $hash{'nodes_h'} = \%nodes;
    $hash{'node_tags_h'} = \%node_tags;
    $hash{'way_tags_h'} = \%way_tags;
    $hash{'way_nd_h'} = \%way_nd;
    $hash{'NODE_a'} = \@NODES;
    $hash{'unbuiltways_a'} = \@unbuiltways;
    $hash{'ways_a'} = \@ways;
    return \%hash;
}

sub set_decimal1_stg($) {
    my $r = shift;
    ${$r} =  int((${$r} + 0.05) * 10) / 10;
    ${$r} = "0.0" if (${$r} == 0);
    ${$r} .= ".0" if !(${$r} =~ /\./);
}

sub set_decimal2_stg($) {
    my $r = shift;
    ${$r} =  int((${$r} + 0.005) * 100) / 100;
    ${$r} = "0.00" if (${$r} == 0);
    ${$r} .= ".00" if !(${$r} =~ /\./);
}
sub set_decimal3_stg($) {
    my $r = shift;
    ${$r} =  int((${$r} + 0.0005) * 1000) / 1000;
    ${$r} = "0.000" if (${$r} == 0);
    ${$r} .= ".000" if !(${$r} =~ /\./);
}

sub set_int_stg($) {
    my $r = shift;
    ${$r} =  int(${$r} + 0.5);
}

sub get_dist_stg_nm($) {
    my ($dist) = @_;
    my $nm = $dist * $SG_METER_TO_NM;
    if ($nm < 1) {
        set_decimal3_stg(\$nm);
    } elsif ($nm < 10) {
        set_decimal2_stg(\$nm);
    } else {
        set_decimal1_stg(\$nm);
    }
    $nm .= "nm";
    return $nm;
}

sub show_ref_hash($) {
    my ($rh) = @_;
    my ($key,$val,$way);
    my ($rtags,$rnds);
    my ($tcnt,$ncnt,$i,$v,$k,$id);
    my ($lat,$lon,$msg,$rll,$cnt,$poly,$tmp);
    foreach $key (keys %{$rh}) {
        $val = ${$rh}{$key};
        prt("$key ");
    }
    prt("\n");
    my $rnodes_h = ${$rh}{'nodes_h'};
    my $rnode_tags_h = ${$rh}{'node_tags_h'};
    my $rway_tags_h = ${$rh}{'way_tags_h'};
    my $rway_nd_h = ${$rh}{'way_nd_h'};
    my $rnode_a = ${$rh}{'NODE_a'};
    my $rubw_a = ${$rh}{'unbuiltways_a'};
    my $ways_a = ${$rh}{'ways_a'};
    $ncnt = scalar keys(%{$rnodes_h});
    prt("Got $ncnt reference points...\n");
    foreach $way (keys %{$rway_tags_h}) {
        if (defined ${$rway_nd_h}{$way}) {
            $poly = "#2D\n"; # is 2D vector
            $rtags = ${$rway_tags_h}{$way};
            $rnds  = ${$rway_nd_h}{$way};
            $tcnt = scalar @{$rtags};
            $ncnt = scalar @{$rnds};
            prt("\nway $way ");
            for ($i = 0; $i < $tcnt; $i++) {
                $v = ${$rtags}[$i][0];
                $k = ${$rtags}[$i][1];
                prt("$v=$k ");
                $poly .= "$v\n" if ($i == 0); # add TYPE
            }
            $poly .= "1\n"; # add only 1 per file
            $poly .= "$ncnt\n"; # number of entries
            $poly .= "0\n"; # whole flag
            prt("$ncnt nd points\n");
            # $nodes{$id} = [$lon,$lat];
            $msg = '';
            $cnt = 0;
            $i = 0;
            my ($llat,$llon,$az1,$az2,$dist);
            my $tot_dist = 0;
            foreach $id (@{$rnds}) {
                $i++;
                if (defined ${$rnodes_h}{$id}) {
                    $rll = ${$rnodes_h}{$id};
                    $lon = ${$rll}[0];
                    $lat = ${$rll}[1];
                    $poly .= sprintf("%.15f  %.15f\n", $lon, $lat);
                    if ($i > 1) {
                        fg_geo_inverse_wgs_84 ($llat,$llon,$lat,$lon,\$az1,\$az2,\$dist);
                        $tot_dist += $dist;
                    }
                    $llat = $lat;
                    $llon = $lon;
                    $lon -= $g_center_lon;
                    $lat -= $g_center_lat;
                    $msg .= "$lat,$lon ";
                    $cnt++;
                    if (($cnt == 3)&&($i < $ncnt)) {
                        $msg .= "\n";
                        $cnt = 0;
                    }
                } else {
                    prtw("WARNING: No REFERNCE point for ID [$id]\n");
                }
            }
            $dist = get_dist_stg_nm($tot_dist);
            prt("Points: $msg $dist\n");
            $poly_count++;
            $tmp = $temp_poly.$poly_count;
            write2file($poly,$tmp);
        } else {
            prtw("WARNING: ID $way in way tags, NOT in way_nd\n");
        }
    }

}

#########################################
### MAIN ###
parse_args(@ARGV);
prt( "$pgmname: in [$cwd]: Hello, World...\n" );
show_ref_hash(process_in_file($in_file));
pgm_exit(0,"Normal exit(0)");
########################################
sub give_help {
    prt("$pgmname: version 0.0.1 2010-09-11\n");
    prt("Usage: $pgmname [options] in-file\n");
    prt("Options:\n");
    prt(" --help (-h or -?) = This help, and exit 0.\n");
}
sub need_arg {
    my ($arg,@av) = @_;
    pgm_exit(1,"ERROR: [$arg] must have following argument!\n") if (!@av);
}

sub parse_args {
    my (@av) = @_;
    my ($arg,$sarg);
    while (@av) {
        $arg = $av[0];
        if ($arg =~ /^-/) {
            $sarg = substr($arg,1);
            $sarg = substr($sarg,1) while ($sarg =~ /^-/);
            if (($sarg =~ /^h/i)||($sarg eq '?')) {
                give_help();
                pgm_exit(0,"Help exit(0)");
            } else {
                pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n");
            }
        } else {
            $in_file = $arg;
            prt("Set input to [$in_file]\n");
        }
        shift @av;
    }

    if ((length($in_file) ==  0) && $debug_on) {
        $in_file = $def_file;
    }
    if (length($in_file) ==  0) {
        pgm_exit(1,"ERROR: No input files found in command!\n");
    }
    if (! -f $in_file) {
        pgm_exit(1,"ERROR: Unable to find in file [$in_file]! Check name, location...\n");
    }
}

# eof - template.pl
