2018-01-16 06:58:15 +01:00
# -*- coding: utf-8 -*-
import base64
import json
import logging
import os
import shutil
import tempfile
import threading
import traceback
from xml . etree import ElementTree as ET
import zipfile
from functools import wraps
from contextlib import closing
from decorator import decorator
import psycopg2
2018-01-16 11:34:37 +01:00
import flectra
from flectra import SUPERUSER_ID
from flectra . exceptions import AccessDenied
import flectra . release
import flectra . sql_db
import flectra . tools
from flectra . sql_db import db_connect
from flectra . release import version_info
from flectra . tools import pycompat
2018-01-16 06:58:15 +01:00
_logger = logging . getLogger ( __name__ )
class DatabaseExists ( Warning ) :
pass
def check_db_management_enabled ( method ) :
def if_db_mgt_enabled ( method , self , * args , * * kwargs ) :
2018-01-16 11:34:37 +01:00
if not flectra . tools . config [ ' list_db ' ] :
2018-01-16 06:58:15 +01:00
_logger . error ( ' Database management functions blocked, admin disabled database listing ' )
raise AccessDenied ( )
return method ( self , * args , * * kwargs )
return decorator ( if_db_mgt_enabled , method )
#----------------------------------------------------------
# Master password required
#----------------------------------------------------------
def check_super ( passwd ) :
2018-01-16 11:34:37 +01:00
if passwd and flectra . tools . config . verify_admin_password ( passwd ) :
2018-01-16 06:58:15 +01:00
return True
2018-01-16 11:34:37 +01:00
raise flectra . exceptions . AccessDenied ( )
2018-01-16 06:58:15 +01:00
2018-01-16 11:34:37 +01:00
# This should be moved to flectra.modules.db, along side initialize().
2018-01-16 06:58:15 +01:00
def _initialize_db ( id , db_name , demo , lang , user_password , login = ' admin ' , country_code = None ) :
try :
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( db_name )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
# TODO this should be removed as it is done by Registry.new().
2018-01-16 11:34:37 +01:00
flectra . modules . db . initialize ( cr )
flectra . tools . config [ ' load_language ' ] = lang
2018-01-16 06:58:15 +01:00
cr . commit ( )
2018-01-16 11:34:37 +01:00
registry = flectra . modules . registry . Registry . new ( db_name , demo , None , update_module = True )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
2018-01-16 11:34:37 +01:00
env = flectra . api . Environment ( cr , SUPERUSER_ID , { } )
2018-01-16 06:58:15 +01:00
if lang :
modules = env [ ' ir.module.module ' ] . search ( [ ( ' state ' , ' = ' , ' installed ' ) ] )
modules . _update_translations ( lang )
if country_code :
countries = env [ ' res.country ' ] . search ( [ ( ' code ' , ' ilike ' , country_code ) ] )
if countries :
env [ ' res.company ' ] . browse ( 1 ) . country_id = countries [ 0 ]
# update admin's password and lang and login
values = { ' password ' : user_password , ' lang ' : lang }
if login :
values [ ' login ' ] = login
2018-01-16 11:34:37 +01:00
emails = flectra . tools . email_split ( login )
2018-01-16 06:58:15 +01:00
if emails :
values [ ' email ' ] = emails [ 0 ]
env . user . write ( values )
cr . execute ( ' SELECT login, password FROM res_users ORDER BY login ' )
cr . commit ( )
except Exception as e :
_logger . exception ( ' CREATE DATABASE failed: ' )
def _create_empty_database ( name ) :
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( ' postgres ' )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
2018-01-16 11:34:37 +01:00
chosen_template = flectra . tools . config [ ' db_template ' ]
2018-01-16 06:58:15 +01:00
cr . execute ( " SELECT datname FROM pg_database WHERE datname = %s " ,
( name , ) , log_exceptions = False )
if cr . fetchall ( ) :
raise DatabaseExists ( " database %r already exists! " % ( name , ) )
else :
cr . autocommit ( True ) # avoid transaction block
cr . execute ( """ CREATE DATABASE " %s " ENCODING ' unicode ' TEMPLATE " %s " """ % ( name , chosen_template ) )
@check_db_management_enabled
def exp_create_database ( db_name , demo , lang , user_password = ' admin ' , login = ' admin ' , country_code = None ) :
""" Similar to exp_create but blocking. """
_logger . info ( ' Create database ` %s `. ' , db_name )
_create_empty_database ( db_name )
_initialize_db ( id , db_name , demo , lang , user_password , login , country_code )
return True
@check_db_management_enabled
def exp_duplicate_database ( db_original_name , db_name ) :
_logger . info ( ' Duplicate database ` %s ` to ` %s `. ' , db_original_name , db_name )
2018-01-16 11:34:37 +01:00
flectra . sql_db . close_db ( db_original_name )
db = flectra . sql_db . db_connect ( ' postgres ' )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
cr . autocommit ( True ) # avoid transaction block
_drop_conn ( cr , db_original_name )
cr . execute ( """ CREATE DATABASE " %s " ENCODING ' unicode ' TEMPLATE " %s " """ % ( db_name , db_original_name ) )
2018-01-16 11:34:37 +01:00
registry = flectra . modules . registry . Registry . new ( db_name )
2018-01-16 06:58:15 +01:00
with registry . cursor ( ) as cr :
# if it's a copy of a database, force generation of a new dbuuid
2018-01-16 11:34:37 +01:00
env = flectra . api . Environment ( cr , SUPERUSER_ID , { } )
2018-01-16 06:58:15 +01:00
env [ ' ir.config_parameter ' ] . init ( force = True )
2018-01-16 11:34:37 +01:00
from_fs = flectra . tools . config . filestore ( db_original_name )
to_fs = flectra . tools . config . filestore ( db_name )
2018-01-16 06:58:15 +01:00
if os . path . exists ( from_fs ) and not os . path . exists ( to_fs ) :
shutil . copytree ( from_fs , to_fs )
return True
def _drop_conn ( cr , db_name ) :
# Try to terminate all other connections that might prevent
# dropping the database
try :
# PostgreSQL 9.2 renamed pg_stat_activity.procpid to pid:
# http://www.postgresql.org/docs/9.2/static/release-9-2.html#AEN110389
pid_col = ' pid ' if cr . _cnx . server_version > = 90200 else ' procpid '
cr . execute ( """ SELECT pg_terminate_backend( %(pid_col)s )
FROM pg_stat_activity
WHERE datname = % % s AND
% ( pid_col ) s != pg_backend_pid ( ) """ % { ' pid_col ' : pid_col},
( db_name , ) )
except Exception :
pass
@check_db_management_enabled
def exp_drop ( db_name ) :
if db_name not in list_dbs ( True ) :
return False
2018-01-16 11:34:37 +01:00
flectra . modules . registry . Registry . delete ( db_name )
flectra . sql_db . close_db ( db_name )
2018-01-16 06:58:15 +01:00
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( ' postgres ' )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
cr . autocommit ( True ) # avoid transaction block
_drop_conn ( cr , db_name )
try :
cr . execute ( ' DROP DATABASE " %s " ' % db_name )
except Exception as e :
_logger . info ( ' DROP DB: %s failed: \n %s ' , db_name , e )
raise Exception ( " Couldn ' t drop database %s : %s " % ( db_name , e ) )
else :
_logger . info ( ' DROP DB: %s ' , db_name )
2018-01-16 11:34:37 +01:00
fs = flectra . tools . config . filestore ( db_name )
2018-01-16 06:58:15 +01:00
if os . path . exists ( fs ) :
shutil . rmtree ( fs )
return True
@check_db_management_enabled
def exp_dump ( db_name , format ) :
with tempfile . TemporaryFile ( mode = ' w+b ' ) as t :
dump_db ( db_name , t , format )
t . seek ( 0 )
return base64 . b64encode ( t . read ( ) ) . decode ( )
@check_db_management_enabled
def dump_db_manifest ( cr ) :
pg_version = " %d . %d " % divmod ( cr . _obj . connection . server_version / 100 , 100 )
cr . execute ( " SELECT name, latest_version FROM ir_module_module WHERE state = ' installed ' " )
modules = dict ( cr . fetchall ( ) )
manifest = {
2018-01-16 11:34:37 +01:00
' flectra_dump ' : ' 1 ' ,
2018-01-16 06:58:15 +01:00
' db_name ' : cr . dbname ,
2018-01-16 11:34:37 +01:00
' version ' : flectra . release . version ,
' version_info ' : flectra . release . version_info ,
' major_version ' : flectra . release . major_version ,
2018-01-16 06:58:15 +01:00
' pg_version ' : pg_version ,
' modules ' : modules ,
}
return manifest
@check_db_management_enabled
def dump_db ( db_name , stream , backup_format = ' zip ' ) :
""" Dump database `db` into file-like object `stream` if stream is None
return a file object with the dump """
_logger . info ( ' DUMP DB: %s format %s ' , db_name , backup_format )
cmd = [ ' pg_dump ' , ' --no-owner ' ]
cmd . append ( db_name )
if backup_format == ' zip ' :
2018-01-16 11:34:37 +01:00
with flectra . tools . osutil . tempdir ( ) as dump_dir :
filestore = flectra . tools . config . filestore ( db_name )
2018-01-16 06:58:15 +01:00
if os . path . exists ( filestore ) :
shutil . copytree ( filestore , os . path . join ( dump_dir , ' filestore ' ) )
with open ( os . path . join ( dump_dir , ' manifest.json ' ) , ' w ' ) as fh :
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( db_name )
2018-01-16 06:58:15 +01:00
with db . cursor ( ) as cr :
json . dump ( dump_db_manifest ( cr ) , fh , indent = 4 )
cmd . insert ( - 1 , ' --file= ' + os . path . join ( dump_dir , ' dump.sql ' ) )
2018-01-16 11:34:37 +01:00
flectra . tools . exec_pg_command ( * cmd )
2018-01-16 06:58:15 +01:00
if stream :
2018-01-16 11:34:37 +01:00
flectra . tools . osutil . zip_dir ( dump_dir , stream , include_dir = False , fnct_sort = lambda file_name : file_name != ' dump.sql ' )
2018-01-16 06:58:15 +01:00
else :
t = tempfile . TemporaryFile ( )
2018-01-16 11:34:37 +01:00
flectra . tools . osutil . zip_dir ( dump_dir , t , include_dir = False , fnct_sort = lambda file_name : file_name != ' dump.sql ' )
2018-01-16 06:58:15 +01:00
t . seek ( 0 )
return t
else :
cmd . insert ( - 1 , ' --format=c ' )
2018-01-16 11:34:37 +01:00
stdin , stdout = flectra . tools . exec_pg_command_pipe ( * cmd )
2018-01-16 06:58:15 +01:00
if stream :
shutil . copyfileobj ( stdout , stream )
else :
return stdout
@check_db_management_enabled
def exp_restore ( db_name , data , copy = False ) :
def chunks ( d , n = 8192 ) :
for i in range ( 0 , len ( d ) , n ) :
yield d [ i : i + n ]
data_file = tempfile . NamedTemporaryFile ( delete = False )
try :
for chunk in chunks ( data ) :
data_file . write ( base64 . b64decode ( chunk ) )
data_file . close ( )
restore_db ( db_name , data_file . name , copy = copy )
finally :
os . unlink ( data_file . name )
return True
@check_db_management_enabled
def restore_db ( db , dump_file , copy = False ) :
assert isinstance ( db , pycompat . string_types )
if exp_db_exist ( db ) :
_logger . info ( ' RESTORE DB: %s already exists ' , db )
raise Exception ( " Database already exists " )
_create_empty_database ( db )
filestore_path = None
2018-01-16 11:34:37 +01:00
with flectra . tools . osutil . tempdir ( ) as dump_dir :
2018-01-16 06:58:15 +01:00
if zipfile . is_zipfile ( dump_file ) :
# v8 format
with zipfile . ZipFile ( dump_file , ' r ' ) as z :
# only extract known members!
filestore = [ m for m in z . namelist ( ) if m . startswith ( ' filestore/ ' ) ]
z . extractall ( dump_dir , [ ' dump.sql ' ] + filestore )
if filestore :
filestore_path = os . path . join ( dump_dir , ' filestore ' )
pg_cmd = ' psql '
pg_args = [ ' -q ' , ' -f ' , os . path . join ( dump_dir , ' dump.sql ' ) ]
else :
# <= 7.0 format (raw pg_dump output)
pg_cmd = ' pg_restore '
pg_args = [ ' --no-owner ' , dump_file ]
args = [ ]
args . append ( ' --dbname= ' + db )
pg_args = args + pg_args
2018-01-16 11:34:37 +01:00
if flectra . tools . exec_pg_command ( pg_cmd , * pg_args ) :
2018-01-16 06:58:15 +01:00
raise Exception ( " Couldn ' t restore database " )
2018-01-16 11:34:37 +01:00
registry = flectra . modules . registry . Registry . new ( db )
2018-01-16 06:58:15 +01:00
with registry . cursor ( ) as cr :
2018-01-16 11:34:37 +01:00
env = flectra . api . Environment ( cr , SUPERUSER_ID , { } )
2018-01-16 06:58:15 +01:00
if copy :
# if it's a copy of a database, force generation of a new dbuuid
env [ ' ir.config_parameter ' ] . init ( force = True )
if filestore_path :
filestore_dest = env [ ' ir.attachment ' ] . _filestore ( )
shutil . move ( filestore_path , filestore_dest )
2018-01-16 11:34:37 +01:00
if flectra . tools . config [ ' unaccent ' ] :
2018-01-16 06:58:15 +01:00
try :
with cr . savepoint ( ) :
cr . execute ( " CREATE EXTENSION unaccent " )
except psycopg2 . Error :
pass
_logger . info ( ' RESTORE DB: %s ' , db )
@check_db_management_enabled
def exp_rename ( old_name , new_name ) :
2018-01-16 11:34:37 +01:00
flectra . modules . registry . Registry . delete ( old_name )
flectra . sql_db . close_db ( old_name )
2018-01-16 06:58:15 +01:00
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( ' postgres ' )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
cr . autocommit ( True ) # avoid transaction block
_drop_conn ( cr , old_name )
try :
cr . execute ( ' ALTER DATABASE " %s " RENAME TO " %s " ' % ( old_name , new_name ) )
_logger . info ( ' RENAME DB: %s -> %s ' , old_name , new_name )
except Exception as e :
_logger . info ( ' RENAME DB: %s -> %s failed: \n %s ' , old_name , new_name , e )
raise Exception ( " Couldn ' t rename database %s to %s : %s " % ( old_name , new_name , e ) )
2018-01-16 11:34:37 +01:00
old_fs = flectra . tools . config . filestore ( old_name )
new_fs = flectra . tools . config . filestore ( new_name )
2018-01-16 06:58:15 +01:00
if os . path . exists ( old_fs ) and not os . path . exists ( new_fs ) :
shutil . move ( old_fs , new_fs )
return True
@check_db_management_enabled
def exp_change_admin_password ( new_password ) :
2018-01-16 11:34:37 +01:00
flectra . tools . config . set_admin_password ( new_password )
flectra . tools . config . save ( )
2018-01-16 06:58:15 +01:00
return True
@check_db_management_enabled
def exp_migrate_databases ( databases ) :
for db in databases :
_logger . info ( ' migrate database %s ' , db )
2018-01-16 11:34:37 +01:00
flectra . tools . config [ ' update ' ] [ ' base ' ] = True
flectra . modules . registry . Registry . new ( db , force_demo = False , update_module = True )
2018-01-16 06:58:15 +01:00
return True
#----------------------------------------------------------
# No master password required
#----------------------------------------------------------
2018-01-16 11:34:37 +01:00
@flectra.tools.mute_logger ( ' flectra.sql_db ' )
2018-01-16 06:58:15 +01:00
def exp_db_exist ( db_name ) :
## Not True: in fact, check if connection to database is possible. The database may exists
try :
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( db_name )
2018-01-16 06:58:15 +01:00
with db . cursor ( ) :
return True
except Exception :
return False
def list_dbs ( force = False ) :
2018-01-16 11:34:37 +01:00
if not flectra . tools . config [ ' list_db ' ] and not force :
raise flectra . exceptions . AccessDenied ( )
2018-01-16 06:58:15 +01:00
2018-01-16 11:34:37 +01:00
if not flectra . tools . config [ ' dbfilter ' ] and flectra . tools . config [ ' db_name ' ] :
# In case --db-filter is not provided and --database is passed, Flectra will not
2018-01-16 06:58:15 +01:00
# fetch the list of databases available on the postgres server and instead will
# use the value of --database as comma seperated list of exposed databases.
2018-01-16 11:34:37 +01:00
res = sorted ( db . strip ( ) for db in flectra . tools . config [ ' db_name ' ] . split ( ' , ' ) )
2018-01-16 06:58:15 +01:00
return res
2018-01-16 11:34:37 +01:00
chosen_template = flectra . tools . config [ ' db_template ' ]
2018-01-16 06:58:15 +01:00
templates_list = tuple ( set ( [ ' postgres ' , chosen_template ] ) )
2018-01-16 11:34:37 +01:00
db = flectra . sql_db . db_connect ( ' postgres ' )
2018-01-16 06:58:15 +01:00
with closing ( db . cursor ( ) ) as cr :
try :
2018-01-16 11:34:37 +01:00
db_user = flectra . tools . config [ " db_user " ]
2018-01-16 06:58:15 +01:00
if not db_user and os . name == ' posix ' :
import pwd
db_user = pwd . getpwuid ( os . getuid ( ) ) [ 0 ]
if not db_user :
2018-01-16 11:34:37 +01:00
cr . execute ( " select usename from pg_user where usesysid=(select datdba from pg_database where datname= %s ) " , ( flectra . tools . config [ " db_name " ] , ) )
2018-01-16 06:58:15 +01:00
res = cr . fetchone ( )
db_user = res and str ( res [ 0 ] )
if db_user :
cr . execute ( " select datname from pg_database where datdba=(select usesysid from pg_user where usename= %s ) and not datistemplate and datallowconn and datname not in %s order by datname " , ( db_user , templates_list ) )
else :
cr . execute ( " select datname from pg_database where not datistemplate and datallowconn and datname not in %s order by datname " , ( templates_list , ) )
2018-01-16 11:34:37 +01:00
res = [ flectra . tools . ustr ( name ) for ( name , ) in cr . fetchall ( ) ]
2018-01-16 06:58:15 +01:00
except Exception :
res = [ ]
res . sort ( )
return res
def list_db_incompatible ( databases ) :
2018-01-16 11:34:37 +01:00
""" " Check a list of databases if they are compatible with this version of Flectra
2018-01-16 06:58:15 +01:00
: param databases : A list of existing Postgresql databases
: return : A list of databases that are incompatible
"""
incompatible_databases = [ ]
server_version = ' . ' . join ( str ( v ) for v in version_info [ : 2 ] )
for database_name in databases :
with closing ( db_connect ( database_name ) . cursor ( ) ) as cr :
2018-01-16 11:34:37 +01:00
if flectra . tools . table_exists ( cr , ' ir_module_module ' ) :
2018-01-16 06:58:15 +01:00
cr . execute ( " SELECT latest_version FROM ir_module_module WHERE name= %s " , ( ' base ' , ) )
base_version = cr . fetchone ( )
if not base_version or not base_version [ 0 ] :
incompatible_databases . append ( database_name )
else :
# e.g. 10.saas~15
local_version = ' . ' . join ( base_version [ 0 ] . split ( ' . ' ) [ : 2 ] )
if local_version != server_version :
incompatible_databases . append ( database_name )
else :
incompatible_databases . append ( database_name )
2018-07-13 11:51:12 +02:00
for database_name in incompatible_databases :
2018-01-16 06:58:15 +01:00
# release connection
2018-01-16 11:34:37 +01:00
flectra . sql_db . close_db ( database_name )
2018-01-16 06:58:15 +01:00
return incompatible_databases
def exp_list ( document = False ) :
2018-01-16 11:34:37 +01:00
if not flectra . tools . config [ ' list_db ' ] :
raise flectra . exceptions . AccessDenied ( )
2018-01-16 06:58:15 +01:00
return list_dbs ( )
def exp_list_lang ( ) :
2018-01-16 11:34:37 +01:00
return flectra . tools . scan_languages ( )
2018-01-16 06:58:15 +01:00
def exp_list_countries ( ) :
list_countries = [ ]
2018-01-16 11:34:37 +01:00
root = ET . parse ( os . path . join ( flectra . tools . config [ ' root_path ' ] , ' addons/base/res/res_country_data.xml ' ) ) . getroot ( )
2018-01-16 06:58:15 +01:00
for country in root . find ( ' data ' ) . findall ( ' record[@model= " res.country " ] ' ) :
name = country . find ( ' field[@name= " name " ] ' ) . text
code = country . find ( ' field[@name= " code " ] ' ) . text
list_countries . append ( [ code , name ] )
return sorted ( list_countries , key = lambda c : c [ 1 ] )
def exp_server_version ( ) :
""" Return the version of the server
Used by the client to verify the compatibility with its own version
"""
2018-01-16 11:34:37 +01:00
return flectra . release . version
2018-01-16 06:58:15 +01:00
#----------------------------------------------------------
# db service dispatch
#----------------------------------------------------------
def dispatch ( method , params ) :
g = globals ( )
exp_method_name = ' exp_ ' + method
if method in [ ' db_exist ' , ' list ' , ' list_lang ' , ' server_version ' ] :
return g [ exp_method_name ] ( * params )
elif exp_method_name in g :
passwd = params [ 0 ]
params = params [ 1 : ]
check_super ( passwd )
return g [ exp_method_name ] ( * params )
else :
raise KeyError ( " Method not found: %s " % method )