Commit 7a1446d7 authored by Felix Delattre's avatar Felix Delattre

initial commit of legacy code

parents
#!/bin/sh
# check for installed OBM tasks:
ls -la /etc/init/obm*.conf
for i in /etc/init/obm*.conf ; do
b=$(basename $i .conf)
echo SRVC $b
service $b stop
service $b status
done
# check for running python processes:
ps -F afwx | grep " [p]ython "
\ No newline at end of file
# Revision: $Id: create-obm-user.sh 1258 2015-08-03 12:54:25Z tyrone $
useradd -c 'OpenBuildingMap general user' -d /home/obm -s /bin/false -m -u 5432 -U obm13
\ No newline at end of file
#!/bin/sh
# Revision: $Id: generate-renderd-layers.sh 1257 2015-08-03 12:53:50Z tyrone $
for t in obm gem ; do
for i in /usr/local/openbuildingmap/${t}-mapnik/${t}*.xml ; do
b=$(basename $i .xml)
# echo START $b
cat <<EOT
[$b]
URI=/tiles/$b/
XML=/usr/local/openbuildingmap/${t}-mapnik/$b.xml
TILEDIR=/var/lib/mod_tile
DESCRIPTION=$b
HOST=127.0.0.1
EOT
done
done
#!/bin/sh
# $Id: shell_snippets 2266 2019-09-05 10:56:58Z tyrone $
_get_path_from_number () {
_tmp=$(printf "%09d" "$1")
_out=''
for line in $(echo "$_tmp" | fold -w 3)
do
_out=${_out}$(printf '%s/' $line)
done
_my_number_string=$(echo $_out | sed 's/\/$//g')
# return $_my_number_string
printf "%s" "$_my_number_string"
}
__bailOut () {
if [ x"$1"x != xx ] ; then
_errMsg="$1"
fi
if [ x"$2"x != xx ] ; then
_errCode=$(echo "$2" | tr -c -d '0-9')
if [ "$2" != "$_errCode" ] ; then
_errCode=0
fi
else
_errCode=0
fi
if [ "$_errCode" -gt 0 ] ; then
__log "$1" 2
else
__log "$1"
fi
exit "$_errCode"
}
__log () {
_msg="$1"
_channel=$(echo "$2" | tr -c -d '0-9')
if [ x"$_msg"x = xx ]
then
return 1
fi
if [ x"$_channel"x = xx ]
then
_channel=0
fi
_msg=$(printf "[%s]\t%s" "$(date -R)" "$_msg")
if [ $_channel = 0 ]
then
echo "$_msg"
elif [ $_channel = 1 ]
then
echo "$_msg"
elif [ $_channel = 2 ]
then
echo "$_msg" >&2
else
return 1
fi
return 0
}
__log "START"
_count=0
for i in $(seq 3311524 3543112)
do
_this_path_component=$(_get_path_from_number $i)
# printf "C:/msdos/system/%s.osc.gz\n" $_this_path_component
_count=$(($_count + 1))
__log "#it: $_count"
done
__log "END"
__bailOut "iterations: $_count"
\ No newline at end of file
wget -m -np -A poly http://download.geofabrik.de/
(see download_poly_files.sh)
perl script from:
https://svn.openstreetmap.org/applications/utils/osm-extract/polygons/poly2wkt.pl
create a WKT-string from a *.poly-file:
./poly2wkt.pl download.geofabrik.de/europe/germany/sachsen.poly
#!/usr/bin/env bash
wget -m -np -A poly http://download.geofabrik.de/
# TODO python script to import the polygons into obm_region table:
# - cleanup table, but if and only if there is no calcClass attached to the region
# - deploy new data into the table
\ No newline at end of file
#!/usr/bin/env python
import re
a = 'download.geofabrik.de/europe/great-britain/england/east-yorkshire-with-hull.poly'
# print a
polyFilePattern = re.compile('^.*download.geofabrik.de/(.*?).poly$')
# dummy = polyFilePattern.match(a).group(0)
dummy = polyFilePattern.match(a).group(0)
# print dummy
data = dummy.split('/')
if len(data) >= 1:
area = data[-1]
print area
if len(data) >= 2:
areaPath = (' ').join(data[1:-1])
print areaPath
else:
print 'no path'
else:
print 'nothing in there...'
#!/usr/bin/env python
import socket
print 'socket.gethostname(): "%s"' % socket.gethostname()
print 'socket.getfqdn(): "%s"' % socket.getfqdn()
ALTER TABLE planet_osm_polygon ADD COLUMN exp_inhabitants float;
ALTER TABLE planet_osm_polygon ADD COLUMN exp_occupancy text;
ALTER TABLE planet_osm_polygon ADD COLUMN exp_occupancy_ot1 text;
ALTER TABLE planet_osm_polygon ADD COLUMN exp_occupancy_ot2 text;
ALTER TABLE planet_osm_polygon ADD COLUMN exp_occupancy_ot3 text;
ALTER TABLE planet_osm_polygon ADD COLUMN exp_position text;
#!/usr/bin/env python
import ConfigParser
import socket
# TODO implement threadsafe logging
# TODO config object
config = ConfigParser.ConfigParser()
configFile = 'exposureConfig.ini'
try:
config.readfp(open(configFile))
except Exception, e:
raise SystemError('cannot open system configuration file: %s' % configFile, e)
else:
print 'Config loaded :)'
# print config.defaults()
# print config.sections()
try:
dsn = config.get(socket.getfqdn(), 'dsn')
except Exception, e:
raise SystemError('cannot find local database connection string', e)
import psycopg2
import psycopg2.extras
import exposureInhabitants
import exposureOccupancy
import exposurePosition
import updateBuilding
# establish database connection; using personal configuration
def processBuildings(dsn):
# TODO database object
try:
DbConnection = psycopg2.connect(dsn)
except Exception, e:
raise RuntimeError('cannot connect to the database', e)
else:
psycopg2.extras.register_hstore(DbConnection, globally=True)
buildingListCursor = DbConnection.cursor(name='1buildingList', cursor_factory=psycopg2.extras.DictCursor)
# TODO seach in a polygon, check for ID style (negative IDs)
# TODO get the LIMIT and/or particular OSM ID(s) from configuration
sqlStatement = """
SELECT osm_id, tags, 0.8 * ST_Area(ST_Transform(way, 32632)) AS building_area
FROM planet_osm_polygon
WHERE tags ? 'building'
;
"""
buildingListCursor.execute(sqlStatement)
# dbconnection/cursor for processing in subroutines:
generalCursor = DbConnection.cursor(cursor_factory=psycopg2.extras.DictCursor)
allBuildingCount = 0
# TODO for-loop in try:/except:/else:
# TODO2 implement threading/message queue
for buildingItem in buildingListCursor:
allBuildingCount += 1
print "START new building: %d (%d)" % (buildingItem[0], allBuildingCount)
gemTaxonomyValues = {}
# TODO generate "obm_"-tags dynamically
gemTaxonomyValues['occupancy'] = exposureOccupancy.calculate(dsn, buildingItem)
gemTaxonomyValues['position'] = exposurePosition.calculate(dsn, buildingItem)
# TODO this should be called for residential buildings only...
# i.e. gemTaxonomyValues['occupancy'] = 'RES*' or 'MIX...'
# TODO into try/except...
gemTaxonomyValues['inhabitants'] = exposureInhabitants.calculate(dsn, buildingItem)
if gemTaxonomyValues['inhabitants'] is None:
gemTaxonomyValues['inhabitants'] = 'NULL'
# update the exposure/taxonomy columns
try:
updateBuilding.update(dsn, gemTaxonomyValues, buildingItem[0])
except Exception, e:
print 'ERROR on UPDATE: ', e
raise SystemError('ERROR: update statement failed')
else:
# everything went fine
pass
# ----------------------------------------------------------------------------------------------------------------------------
# we're done with the building:
print ''
buildingListCursor.close()
DbConnection.close()
print "END - #buildings: %d :)" % allBuildingCount
# select all buildings in a polygon - we use Berlin
if __name__ == "__main__":
# TODO add time check
processBuildings(dsn)
#!/bin/sh
# TODO get DB name from config
for i in point line polygon ; do
echo "TRUNCATE TABLE planet_osm_$i;" ;
pg_dump -O -x -t planet_osm_$i --column-inserts -a testberlintaxonomy
done | gzip > /tmp/taxDB.sql.gz
# simple ini style configuration file for exposure stuff...
# ...not as powerful as JSON/YAML/XML but for a PoC ok ;)
#
# host specific settings are in the sections named after the host fqdn(!).
# use ./_show_hostname.py to show python's opinion about that on your system
#
# global options should be set in the "DEFAULT" section
#
# values in host sections overwrite values in the "DEFAULT" section
# default section, required!
[DEFAULT]
joe = cool
atze = lame
debug = 0
# dummy section for tetsing^Htesting
[adresse.geht.net]
dsn = XY
# exposure VM on kodama:
[exposure.kodama.vpn]
dsn: dbname=berlintax
# kodama:
[kodama.quake.gfz-potsdam.de]
dsn: dbname=testberlintaxonomy
# tb's notebook:
[phantom.bug.local]
dsn: dbname=berlin_exposure
debug = 2
building_id = 12345678
building_count_limit = 1000
# Danijel's notebook:
[sputnik.schorlemmer.net]
dsn: dbname=berlinsw
#!/usr/bin/env python
"""
Estimation of inhabitants in a building where the #levels is available
"""
# TODO this should be called for residential buildings only...
def calculate(dsn, buildingLine):
if 'building:levels' in buildingLine[1]:
# TODO calculate the inhabitants
try:
levels = float(buildingLine[1]['building:levels'])
except Exception, e:
# TODO log invalid content in building:levels
print ' WARNING: invalid content in building:levels =>%s<=' % buildingLine[1]['building:levels']
else:
return buildingLine[2] * levels / 38.4
return None
else:
return None
if __name__ == '__main__':
raise RuntimeError('You must not call this separately!')
This diff is collapsed.
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
Estimation of inhabitants in a building where the #levels is available
"""
import math
import psycopg2
import psycopg2.extras
# TODO this should be called for residential buildings only...
def calculate(dsn, buildingLine):
try:
positionConnection = psycopg2.connect(dsn)
except Exception, e:
raise RuntimeError('landuse: cannot connect to the database', e)
positionCursor = positionConnection.cursor(cursor_factory=psycopg2.extras.DictCursor)
# TODO the dimension of the intersetion must be at least 1
sqlStatement = """
SELECT DISTINCT ON (p.osm_id) p.osm_id,
ST_Dimension(ST_Intersection(p.way, b.way)) AS dimension
FROM planet_osm_polygon AS p
JOIN planet_osm_polygon AS b
ON ( ST_Intersects(p.way, b.way) )
WHERE b.osm_id = %d AND p.tags ? 'building'
AND b.tags ? 'building' AND p.osm_id <> %d AND NOT ST_Equals(p.way, b.way)
AND ST_Dimension(ST_Intersection(p.way, b.way)) > 0;
""" % (buildingLine[0], buildingLine[0])
try:
positionCursor.execute(sqlStatement)
except Exception, e:
print ' WARNING: cannot check position because of exception: ', e
else:
print ' BUILDING_POSITION DEBUG: found %d rows for position' % positionCursor.rowcount
if positionCursor.rowcount < 0:
raise SystemError('ERROR in position query: ', positionCursor.statusmessage)
else:
numAdjBuildings = min(5, positionCursor.rowcount)
positionMap = ['BPD', 'BP1', 'BP2', 'BP3', 'BP4', 'BP5']
if numAdjBuildings == 2:
# TODO
print ' BUILDING_POSITION: %d has two adjacent buildings' % buildingLine[0]
try:
return _bp2ab(dsn, buildingLine[0])
except ValueError, e:
print ' WARNING: ', e
return 'BP2'
except Exception, e:
raise SystemError('ERROR: two adj. buildings: ', e)
else:
return positionMap[numAdjBuildings]
finally:
try:
positionCursor.close()
except Exception, e:
# something wrong, but we do not really care...
pass
positionConnection.close()
def _bp2ab(dsn, buildingId):
# ok, we could just use the result from the main function, so we don't had to create a new connection, but...
try:
__twoAdjBuildConnection = psycopg2.connect(dsn)
except Exception, e:
raise RuntimeError('landuse: cannot connect to the database', e)
__twoAdjBuildCursor = __twoAdjBuildConnection.cursor(cursor_factory=psycopg2.extras.DictCursor)
sqlStatement = """
SELECT osm_id, dimension, type, point, text_point
FROM (
SELECT 0 AS type,
b.osm_id,
0 AS dimension,
ST_AsText(ST_Transform(ST_Centroid(b.way), 32632)) AS text_point,
ST_Transform(ST_Centroid(b.way), 32632) AS point
FROM planet_osm_polygon AS b
WHERE b.osm_id = %d
UNION
SELECT DISTINCT ON (p.osm_id)
1 AS type,
p.osm_id,
ST_Dimension(ST_Intersection(p.way, b.way)) AS dimension,
ST_AsText(ST_Transform(ST_Centroid(p.way), 32632)) AS text_point,
ST_Transform(ST_Centroid(p.way), 32632) AS point
FROM planet_osm_polygon AS p
JOIN planet_osm_polygon AS b
ON ( ST_Intersects(p.way, b.way) )
WHERE b.osm_id = %d
AND p.osm_id <> %d
AND ST_Distance(ST_Centroid(p.way), ST_Centroid(b.way)) > 0
AND p.tags ? 'building'
AND b.tags ? 'building'
) AS foo
ORDER BY type ASC, dimension DESC;
""" % (buildingId, buildingId, buildingId)
try:
__twoAdjBuildCursor.execute(sqlStatement)
except Exception, e:
print 'DEBUG DB query: %s' % sqlStatement
raise ValueError('cannot check angel for two adjeacent buildings because of exception: ', e)
else:
if __twoAdjBuildCursor.rowcount < 0:
raise SystemError('ERROR in two adjacent buildings query: ', __twoAdjBuildCursor.statusmessage)
else:
# if result contains more than three items something went wrong...
if __twoAdjBuildCursor.rowcount != 3: