geo/cron/traite_osm.sh

#!/bin/bash

ladate=$(date +%Y%m%d-%H%M)
lheure=$(date +%H:%M:%S)

mybase=osm
myuser=osm_maj_user
mypass=d66,OsmMaj.User

log=osm.log
rm -f $log
touch $log
echo "To: contact@d6soft.fr" >> $log
echo "From: contact@d6soft.fr" >> $log
echo "Subject: [Import] OSM dva-maria - $ladate" >> $log

continent=australia-oceania
country=new-caledonia

cd /var/osm

echo "Demarrage du telechargement de la Nouvelle-Caledonie a $lheure" >> $log

rm -f ${country}-latest.osm.pbf
rm -f ${country}.o5m
rm -f ${country}-streets.csv
rm -f streets.csv

lheure=$(date +%H:%M:%S)
echo "Debut du telechargement de la Nouvelle-Caledonie a $lheure" >> $log
wget http://download.geofabrik.de/${continent}/${country}-latest.osm.pbf -O ${country}-latest.osm.pbf
if [ "$?" -ne "0" ] ; then
       	echo "!!!!  Désolé, le téléchargement deNouvelle-Caledonie semble ne pas fonctionner. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $log
        # exit 1
else	
	lheure=$(date +%H:%M:%S)
	echo "Telechargement de la Nouvelle-Caledonie terminee a $lheure" >> $log

	osmconvert -v ${country}-latest.osm.pbf --all-to-nodes --drop-relations --max-objects=99999999 --out-o5m > ${country}.o5m
	lheure=$(date +%H:%M:%S)
	echo "Generation du fichier .o5m terminee a $lheure" >> $log

	osmconvert ${country}.o5m --csv="@id @lat @lon name place addr:street addr:housenumber addr:city addr:village addr:town" --csv-headline --csv-separator=, -o=${country}-streets.csv
	lheure=$(date +%H:%M:%S)
	echo "Conversion en .csv terminee a $lheure" >> $log

	awk -F, '$4 != "" || $5 != "" || $6 != "" || $8 != ""' ${country}-streets.csv > streets.csv
	lheure=$(date +%H:%M:%S)
	echo "Filtrage du csv termine a $lheure" >> $log

	table="${country//-/_}"

        rm -f cde.sql
       	touch cde.sql
        echo "USE $mybase;" >> cde.sql
       	echo "DROP TABLE IF EXISTS $table;" >> cde.sql
	echo "CREATE TABLE $table (rowid int(11) NOT NULL AUTO_INCREMENT, osm_id int(11) DEFAULT NULL, osm_lat varchar(10) DEFAULT NULL, osm_lng varchar(10) DEFAULT NULL, osm_name varchar(60) DEFAULT NULL, osm_place varchar(60) DEFAULT NULL, osm_street varchar(60) DEFAULT NULL, osm_number varchar(10) DEFAULT NULL, osm_city varchar(60) DEFAULT NULL, osm_date_creat timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (rowid)) ENGINE=InnoDB DEFAULT CHARSET=utf8;" >> cde.sql
	echo "ALTER TABLE $table ADD INDEX osm_lat (osm_lat ASC), ADD INDEX osm_lng (osm_lng ASC);" >> cde.sql
	echo "LOAD DATA LOCAL INFILE 'streets.csv' INTO TABLE $table CHARACTER SET 'UTF8mb4' FIELDS TERMINATED BY ',' IGNORE 1 LINES (osm_id, osm_lat, osm_lng, osm_name, osm_place, osm_street, osm_number, osm_city);" >> cde.sql

	mariadb --local-infile=1 -u$myuser -p$mypass $mybase < cde.sql
	lheure=$(date +%H:%M:%S)
	echo "Importation sql de la Nouvelle-Caledonie dans la table $table terminee a $lheure" >> $log

	rm -f ${country}-latest.osm.pbf
	rm -f ${country}.o5m
	rm -f ${country}-streets.csv
	rm -f streets.csv
fi


continent=europe
country=france

cd /var/osm

echo "Demarrage du telechargement des regions a $lheure" >> $log

for region in alsace aquitaine auvergne basse-normandie bourgogne bretagne centre champagne-ardenne corse franche-comte guadeloupe guyane haute-normandie ile-de-france languedoc-roussillon limousin lorraine martinique mayotte midi-pyrenees nord-pas-de-calais pays-de-la-loire picardie poitou-charentes provence-alpes-cote-d-azur reunion rhone-alpes 
do
	rm -f ${country}-${region}-latest.osm.pbf
	rm -f ${country}-${region}.o5m
	rm -f ${country}-${region}-streets.csv
	rm -f *.csv

	lheure=$(date +%H:%M:%S)
	echo "Debut du telechargement de la region $region a $lheure" >> $log
	wget http://download.geofabrik.de/${continent}/${country}/${region}-latest.osm.pbf -O ${country}-${region}-latest.osm.pbf
	if [ "$?" -ne "0" ] ; then
        	echo "!!!!  Désolé, le téléchargement de la carte $region semble ne pas fonctionner. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >> $log
	        # exit 1
	else	
		lheure=$(date +%H:%M:%S)
		echo "Telechargement de la region $region terminee a $lheure" >> $log

		osmconvert -v ${country}-${region}-latest.osm.pbf --all-to-nodes --drop-relations --max-objects=99999999 --out-o5m > ${country}-${region}.o5m
		lheure=$(date +%H:%M:%S)
		echo "Generation du fichier .o5m terminee a $lheure" >> $log

		osmconvert ${country}-${region}.o5m --csv="@id @lat @lon name place addr:street addr:housenumber addr:city addr:village addr:town" --csv-headline --csv-separator=, -o=${country}-${region}-streets.csv
		lheure=$(date +%H:%M:%S)
		echo "Conversion en .csv terminee a $lheure" >> $log

		awk -F, '$4 != "" || $5 != "" || $6 != "" || $8 != ""' ${country}-${region}-streets.csv > streets.csv
		lheure=$(date +%H:%M:%S)
		echo "Filtrage du csv termine a $lheure" >> $log

		split -l 1000 streets.csv streets_1000_

		table="${region//-/_}"

	        rm -f cde.sql
        	touch cde.sql
	        echo "USE $mybase;" >> cde.sql
        	echo "DROP TABLE IF EXISTS $table;" >> cde.sql
		echo "CREATE TABLE $table (rowid int(11) NOT NULL AUTO_INCREMENT, osm_id int(11) DEFAULT NULL, osm_lat varchar(10) DEFAULT NULL, osm_lng varchar(10) DEFAULT NULL, osm_name varchar(60) DEFAULT NULL, osm_place varchar(60) DEFAULT NULL, osm_street varchar(60) DEFAULT NULL, osm_number varchar(10) DEFAULT NULL, osm_city varchar(60) DEFAULT NULL, osm_date_creat timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (rowid)) ENGINE=InnoDB DEFAULT CHARSET=utf8;" >> cde.sql
		echo "ALTER TABLE $table ADD INDEX osm_lat (osm_lat ASC), ADD INDEX osm_lng (osm_lng ASC);" >> cde.sql
		mysql --local-infile=1 -u$myuser -p$mypass $mybase < cde.sql
		lheure=$(date +%H:%M:%S)
		echo "Creation sql de la region $region $a dans la table $table terminee a $lheure" >> $log

		for a in $(find . -name "streets_1000_*" | sort);
		do
	        	rm -f cde.sql
	        	touch cde.sql
		        echo "USE $mybase;" >> cde.sql
			echo "LOAD DATA LOCAL INFILE '${a}' INTO TABLE $table CHARACTER SET 'UTF8mb4' FIELDS TERMINATED BY ',' IGNORE 1 LINES (osm_id, osm_lat, osm_lng, osm_name, osm_place, osm_street, osm_number, osm_city);" >> cde.sql

			mariadb --local-infile=1 -u$myuser -p$mypass $mybase < cde.sql
			lheure=$(date +%H:%M:%S)
			echo "Importation sql de %a pour la region $region $a dans la table $table terminee a $lheure" >> $log
		done
		
		rm -f ${country}-${region}-latest.osm.pbf
		rm -f ${country}-${region}.o5m
		rm -f ${country}-${region}-streets.csv
		rm -f streets_1000_*
	        rm -f *.csv
	fi
done


lheure=$(date +%H:%M:%S)
echo "Fin des traitements OSM a $lheure" >> $log

ssmtp support@unikoffice.com < $log