#!/usr/bin/python
##############################################################################
# Copyright (c) Members of the EGEE Collaboration. 2004.
# See http://www.eu-egee.org/partners/ for details on the copyright
# holders.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at #
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##############################################################################

import os
import re
import sys
import time
import getopt
import socket
import signal
import string
import pickle
import logging
import tempfile

def parse_options():

    config = {}

    try:
        opts, args = getopt.getopt(sys.argv[1:], "dc:", ["config"])
    except getopt.GetoptError:
        sys.stderr.write("Error: Invalid option specified.\n")
        print_usage()
        sys.exit(2)
    for o, a in opts:
        if o in ("-c", "--config"):
            config['BDII_CONFIG_FILE'] = a
        if o in ("-d", "--daemon"):
            config['BDII_DAEMON'] = True

    if ( not config.has_key('BDII_CONFIG_FILE')):
        sys.stderr.write("Error: Configuration file not specified.\n")
        print_usage()
        sys.exit(1)         

    if ( not os.path.exists(config['BDII_CONFIG_FILE']) ):
        sys.stderr.write("Error: Configuration file %s does not exist.\n" %  config['BDII_CONFIG_FILE'])
        sys.exit(1)         

    return config

def get_config(config):
    
    for line in open(config['BDII_CONFIG_FILE']).readlines():
        index=line.find("#")
        if (index > -1):
            line=line[:index]
        index=line.find("=")
        if (index > -1):
            key=line[:index].strip()
            value=line[index+1:].strip()
            config[key] = value

    if 'SLAPD_CONF' in os.environ:
        config['SLAPD_CONF'] = os.environ['SLAPD_CONF']

    if ( not config.has_key('BDII_DAEMON') ):
        config['BDII_DAEMON'] = False

    if ( not config.has_key('BDII_RUN_DIR') ):
        config['BDII_RUN_DIR'] = '/var/run/bdii'

    if ( not config.has_key('BDII_PID_FILE') ):
        config['BDII_PID_FILE'] = "%s/bdii-update.pid" % config['BDII_RUN_DIR']

    for parameter in ['BDII_LOG_FILE', 'BDII_LOG_LEVEL', 'BDII_LDIF_DIR',
                      'BDII_PROVIDER_DIR', 'BDII_PLUGIN_DIR',
                      'BDII_READ_TIMEOUT']:
        if ( not config.has_key(parameter) ):
            sys.stderr.write("Error: Configuration parameter %s is not specified in the configuration file %s.\n" % (parameter, config['BDII_CONFIG_FILE']))
            sys.exit(1)

    for parameter in ['BDII_LDIF_DIR','BDII_PROVIDER_DIR','BDII_PLUGIN_DIR']:
        if ( not os.path.exists(config[parameter])):
            sys.stderr.write("Error: %s %s does not exist.\n" %  (parameter, config[parameter]))
            sys.exit(1)
    if not config.has_key('BDII_LOG_LEVEL'):
        config['BDII_LOG_LEVEL']='ERROR'
    else:
        log_levels=['CRITICAL','ERROR','WARNING','INFO','DEBUG']
        try:
            log_levels.index(config['BDII_LOG_LEVEL'])
        except ValueError, e:
            sys.stderr.write("Error: Log level %s is not an allowed level. %s\n" % (config['BDII_LOG_LEVEL'], log_levels))
            sys.exit(1)

    config['BDII_READ_TIMEOUT'] = int(config['BDII_READ_TIMEOUT'])

    if ( config['BDII_DAEMON'] == True ):
        for parameter in ['BDII_PORT', 'BDII_BREATHE_TIME', 'BDII_VAR_DIR',
                          'BDII_ARCHIVE_SIZE', 'BDII_DELETE_DELAY',
                          'SLAPD_CONF']:
            if ( not config.has_key(parameter) ):
                sys.stderr.write("Error: Configuration parameter %s is not specified in the configuration file %s.\n" % (parameter, config['BDII_CONFIG_FILE']))
                sys.exit(1)
        if ( os.path.exists(config['SLAPD_CONF']) ):
            config['BDII_PASSWD'] = {}
            config['BDII_PASSWD_FILE'] = {}
            if not os.path.exists(config['BDII_RUN_DIR']):
                os.makedirs(config['BDII_RUN_DIR'])
            rootdn = False
            rootpw = False
            filename = ""
            for line in open(config['SLAPD_CONF']):
                if ( line.find("rootdn") > -1 ):
                    rootdn = line.replace("rootdn","").strip()
                    rootdn = rootdn.replace('"','').replace(" ","")
                    filename = rootdn.replace('o=','')
                    if ( rootpw ):
                        config['BDII_PASSWD'][rootdn] = rootpw
                        config['BDII_PASSWD_FILE'][rootdn] = "%s/%s" % (config['BDII_RUN_DIR'], filename) 
                        pf = os.open(config['BDII_PASSWD_FILE'][rootdn], os.O_WRONLY | os.O_CREAT, 0600)
                        os.write(pf,rootpw) 
                        os.close(pf) 
                        rootdn = False
                        rootpw = False
                if ( line.find("rootpw") > -1 ):
                    rootpw = line.replace("rootpw","").strip()
                    if ( rootdn ):
                        config['BDII_PASSWD'][rootdn] = rootpw
                        config['BDII_PASSWD_FILE'][rootdn] = "%s/%s" % (config['BDII_RUN_DIR'], filename)
                        pf = os.open(config['BDII_PASSWD_FILE'][rootdn], os.O_WRONLY | os.O_CREAT, 0600)
                        os.write(pf,rootpw)
                        os.close(pf)
                        rootdn = False
                        rootpw = False
        config['BDII_BREATHE_TIME'] = float(config['BDII_BREATHE_TIME'])
        config['BDII_ARCHIVE_SIZE'] = int(config['BDII_ARCHIVE_SIZE'])
        config['BDII_DELETE_DELAY'] = int(config['BDII_DELETE_DELAY'])
        config['BDII_HOSTNAME'] = 'localhost'

    return config

def print_usage():
    sys.stderr.write('''Usage: %s [ OPTIONS ] 
 
     -c --config      BDII configuration file
     -d --daemon      Run BDII in daemon mode
''' % (str(sys.argv[0])))

def create_daemon(log_file):
    try:
        pid = os.fork()
    except OSError, e:
        return((e.errno, e.strerror))

    if (pid == 0):
        os.setsid()
        signal.signal(signal.SIGHUP, signal.SIG_IGN)
        try:
            pid = os.fork()
        except OSError, e:
            return((e.errno, e.strerror))
        if (pid == 0):
            os.umask(022)
        else:
            os._exit(0)
    else:
        os._exit(0)
    try:
        maxfd=os.sysconf("SC_OPEN_MAX")
    except (AttributeError, ValueError):
        maxfd=256
    for fd in range(3, maxfd):
        try:
            os.close(fd)
        except OSError:
            pass
    os.close(0)
    os.open("/dev/null", os.O_RDONLY)
    os.close(1)
    os.open("/dev/null", os.O_WRONLY)

    # connect stderr to log file
    e = os.open(log_file, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0644)
    os.dup2(e, 2)
    os.close(e)
    sys.stderr = os.fdopen(2, 'a', 0)

    # Write PID
    pid_file = open(config['BDII_PID_FILE'],'w')
    pid_file.write("%s\n" % (str(os.getpid())))
    pid_file.close()

def get_logger(log_file,log_level):

    log = logging.getLogger('bdii-update')
    hdlr = logging.StreamHandler(sys.stderr)
    formatter = logging.Formatter('%(asctime)s: [%(levelname)s] %(message)s')
    hdlr.setFormatter(formatter)
    log.addHandler(hdlr)
    log.setLevel(logging.__dict__.get(log_level))

    return log

def handler(signum, frame):
    if ( signum ==14 ):
        # Commit suicide
        process_group=os.getpgrp()
        os.killpg(process_group, signal.SIGTERM)
        sys.exit(1)

def read_ldif(source):

    # Get pipe file descriptors
    read_fd, write_fd = os.pipe()

    # Fork
    pid = os.fork()

    if pid:
        
        # Close write file descriptor as we don't need it.
        os.close(write_fd)
     
        read_fh = os.fdopen(read_fd)
        raw_ldif = read_fh.read()
        result = os.waitpid(pid, 0)
        if (result[1] > 0):
            log.error("Timed out while reading %s", (source))
            return ""
        raw_ldif = raw_ldif.replace("\n ", "")
        
        return raw_ldif

    else:
        
        # Close read file d
        os.close(read_fd)
        
        # Set process group
        os.setpgrp()
                
        # Setup signal handler
        signal.signal(signal.SIGALRM, handler)
        signal.alarm(config['BDII_READ_TIMEOUT'])

        # Open pipe to LDIF
        if ( source[:7] == 'ldap://'):
            url=source.split('/')
            command = "ldapsearch -LLL -x -h %s -b %s 2>/dev/null" % ( url[2], url[3])
            pipe = os.popen(command)
        elif( source[:7] == 'file://' ):
            pipe=open(source[7:])
        else:
            pipe=os.popen(source)    

        raw_ldif=pipe.read()

        # Close LDIF pipe
        pipe.close()
        try:
            write_fh = os.fdopen(write_fd, 'w')
            write_fh.write(raw_ldif)
            write_fh.close()
        except IOError:
            log.error("Information provider %s terminated unexpectedly." % source)
        signal.alarm(0) # Disable the alarm
        sys.exit(0)

def get_dns(ldif):
    dns = {}
    last_dn_index = len(ldif)
    while ( 1 ):
        dn_index = ldif.rfind("dn:",0,last_dn_index)
        if ( dn_index == -1):
            break
        end_dn_index = ldif.find("\n",dn_index, last_dn_index) 
        dn = ldif[dn_index + 4 :end_dn_index].lower()
        dn = re.sub("\s*,\s*",",",dn)
        dn = re.sub("\s*=\s*","=",dn)
        dn = dn.replace("\\5c","\\\\") # Replace encoded slash
        dn = dn.replace("\\2c","\\,") # Replace encoded comma
        dn = dn.replace("\\3d","\\=") # Replace encoded equals
        dn = dn.replace("\\2b","\\+") # Replace encoded plus
        dn = dn.replace("\\3b","\\;") # Replace encoded semi colon
        dn = dn.replace("\\22","\\\"") # Replace encoded quote
        dn = dn.replace("\\3e","\\>") # Replace encoded greater than
        dn = dn.replace("\\3c","\\<") # Replace encoded less than
        end_entry_index = ldif.find("\n\n",dn_index, last_dn_index) 
        dns[dn] = (dn_index, last_dn_index, end_dn_index)
        last_dn_index = dn_index
    return dns

def group_dns(dns):
    grouped = {}
    for dn in dns:
        index = dn.rfind(",")
        root = dn[index +1 :].strip()
        if grouped.has_key(root):
            grouped[root].append(dn)
        else:
            if root in config['BDII_PASSWD']:
                grouped[root] = [ dn ]
            else:
                if "o=shadow" in config['BDII_PASSWD'] and root == "o=grid":
                    grouped[root] = [ dn ]
                elif root != "o=shadow":
                    log.error("dn suffix %s in not specified in the slapd configuration file." % (root))


    return grouped

def convert_entry(entry_string):

    multivalued = [ 'objectclass',
                    'gluehostapplicationsoftwareruntimeenvironment',
                    'glueserviceaccesscontrolrule',
                    'glueserviceaccesscontrolbaserule',
                    'glueceaccesscontrolbaserule',
                    'gluesaaccesscontrolbaserule',
                    'gluesecontrolprotocolcapability',
                    'glueseaccessprotocolsupportedsecurity',
                    'gluesacapability',
                    'gluevoinfoaccesscontrolbaserule',
                    'gluesecontrolprotocolcapability',
                    'gluecesebindgroupseuniqueid',
                    'gluecesebindseuniqueid',
                    'gluecesebindmountinfo',
                    'gluecesebindweight',
                    'glueserviceowner',
                    'gluechunkkey',
                    'glueforeignkey',
                    'gluecesebindmountinfo',
                    'glueseaccessprotocolcapability',
                    'gluesiteotherinfo',
                    'gluesitesponsor',
                    'gluececapability', 
                    'glueclusterservice',
                    'glue2entityotherinfo',
                    'glue2extensionentityforeignkey',
                    'glue2locationserviceforeignkey',
                    'glue2locationdomainforeignkey',
                    'glue2contactserviceforeignkey',
                    'glue2contactdomainforeignkey',
                    'glue2domainwww',
                    'glue2admindomainowner',
                    'glue2admindomainadmindomainforeignkey',
                    'glue2userdomainusermanager',
                    'glue2userdomainmember',
                    'glue2userdomainuserdomainforeignkey',
                    'glue2servicecapability',
                    'glue2servicestatusinfo',
                    'glue2serviceadmindomainforeignkey',
                    'glue2serviceserviceforeignkey',
                    'glue2endpointcapability',
                    'glue2endpointinterfaceextension',
                    'glue2endpointwsdl',
                    'glue2endpointsupportedprofile',
                    'glue2endpointsemantics',
                    'glue2endpointtrustedca',
                    'glue2shareendpointforeignkey',
                    'glue2shareresourceforeignkey',
                    'glue2activityactivityforeignkey',
                    'glue2policyrule',
                    'glue2policyuserdomainforeignkey',
                    'glue2accesspolicyendpointforeignkey',
                    'glue2mappingpolicyshareforeignkey',
                    'glue2computingendpointjobdescription',
                    'glue2computingsharetag',
                    'glue2computingsharecomputingendpointforeignkey',
                    'glue2computingshareexecutionenvironmentforeignkey',
                    'glue2computingmanagernetworkinfo',
                    'glue2executionenvironmentnetworkinfo',
                    'glue2applicationenvironmentbestbenchmark',
                    'glue2applicationenvironmentexecutionenvironmentforeignkey',
                    'glue2computingactivitystate',
                    'glue2computingactivityrestartstate',
                    'glue2computingactivityrequestedapplicationenvironment',
                    'glue2computingactivityexecutionnode',
                    'glue2storageshareaccessmode',
                    'glue2storageshareretentionpolicy',
                    'glue2storagesharestorageendpointforeignkey',
                    'glue2storagesharedatastoreforeignkey',
                    'glue2storagesharecapacitystorageshareforeignkey',
                    'glue2tocomputingservicestorageaccessprotocolforeignkey']
                    
    entry = {}
    for line in entry_string.split("\n"):
        index = line.find(":")
        if (index > -1):
            attribute = line[:index].lower()
            value = line[index + 1:].strip()
            
            if entry.has_key(attribute):
                if not value in entry[attribute]:
                    entry[attribute].append(value)
            else:
                entry[attribute] = [value]
    return entry

def convert_back(entry):
    entry_string =  "dn: %s\n" %(entry["dn"][0])
    entry.pop("dn")
    for attribute in entry.keys():
        attribute=attribute.lower()
        for value in entry[attribute]:
            entry_string += "%s: %s\n" %(attribute, value)

    return entry_string

def ldif_diff(dn, old_entry, new_entry):

    add_attribute={}
    delete_attribute={}
    replace_attribute={}
    
    old_entry = convert_entry(old_entry)
    new_entry = convert_entry(new_entry)
    dn_perserved_case = None
    for attribute in new_entry.keys():
        attribute=attribute.lower()
        if (attribute == "dn"):
            dn_perserved_case = new_entry['dn'][0]
            continue

        # If the old entry has the attribue we need to compare values 
        if ( old_entry.has_key(attribute) ):

            # If the old entries are different find the modify. 
            if ( not new_entry[attribute] == old_entry[attribute]):
                    replace_attribute[attribute] = new_entry[attribute]

        # The old entry does not have the attribute so add it. 
        else:
            add_attribute[attribute] =  new_entry[attribute]

    # Checking for removed attributes
    for attribute in old_entry.keys():
        if (attribute.lower() == "dn"):
            continue
        if not new_entry.has_key(attribute):
            delete_attribute[attribute]=old_entry[attribute]


    # Create LDIF modify statement
    ldif=['dn: %s' % dn_perserved_case ]
    ldif.append('changetype: modify')
    for attribute in add_attribute.keys():
        attribute=attribute.lower()
        ldif.append('add: %s' % (attribute) )
        for value in add_attribute[attribute]:
            ldif.append('%s: %s' % (attribute, value))
        ldif.append('-')
    for attribute in replace_attribute.keys():
        attribute=attribute.lower()
        ldif.append('replace: %s' % (attribute) )
        for value in replace_attribute[attribute]:
            ldif.append('%s: %s' % (attribute, value))
        ldif.append('-')
    for attribute in delete_attribute.keys():
        attribute=attribute.lower()
        ldif.append('delete: %s' % (attribute) )
        ldif.append('-')

    if (len(ldif) > 3):
        ldif = "\n".join(ldif) + "\n\n"
    else:
        ldif = ""
        
    return ldif

def modify_entry(entry, mods):
    mods = convert_entry(mods)
    entry = convert_entry(entry)
    if ( mods.has_key('changetype')):
        # Handle LDIF delete attribute
        if ( mods.has_key('delete')):
            for attribute in mods['delete']:
                attribute=attribute.lower()
                if (entry.has_key(attribute)):
                    if (mods.has_key(attribute)):
                        for value in mods[attribute]:
                            try:
                                entry[attribute].remove(value)
                                if (len(entry[attribute]) == 0):
                                    entry.pop(attribute)
                            except ValueError, e:
                                pass
                            except KeyError, e:
                                pass
                    else:
                        entry.pop(attribute)

        # Handle LDIF replace attribute
        if ( mods.has_key('replace')):
            for attribute in mods['replace']:
                attribute=attribute.lower()
                if (entry.has_key(attribute)):
                    if (mods.has_key(attribute)):
                        entry[attribute] = mods[attribute]

        # Handle LDIF add attribute
        if ( mods.has_key('add')):
            for attribute in mods['add']:
                attribute=attribute.lower()
                if ( not entry.has_key(attribute)):
                    log.debug("attribute: %s" %(attribute))
                    entry[attribute] = mods[attribute]
                else:
                    entry[attribute].extend(mods[attribute])
                    
    # Just old style just change
    else:
        for attribute in mods.keys():
            if (entry.has_key(attribute)):
                entry[attribute]=mods[attribute]

    entry_string = convert_back(entry)
    return entry_string
    
def fix(dns,ldif):
    response = []
    append = response.append
    
    for dn in dns.keys():
        entry = convert_entry(ldif[dns[dn][0]:dns[dn][1]])
        if  ( dn[:11].lower() == "mds-vo-name" ):
            if 'objectclass' in entry:
                if 'mds' in map( lambda x : x.lower(), entry['objectclass']):
                    if 'gluetop' in map( lambda x : x.lower(), entry['objectclass']):
                        value=dn[12:dn.index(",")]
                        entry = { 'dn': [dn], 'objectclass': ['MDS'], 'mds-vo-name': [value] }
        entry = convert_back(entry)         
        append(entry)
    response = "".join(response)
    return response

def log_errors(error_file, dns):    
    log.debug("Logging Errors")
    request=0
    dn = None
    error_counter=0
    for line in open(error_file).readlines():
        if ( line[:7] == 'request' ):
            request += 1
        else:
            if ( request > 1 ):
                try:
                    if ( not dn == dns[request - 2] ):
                        error_counter += 1
                        dn = dns[request - 2]
                        log.warn( "dn: %s" %(dn) )
                except IndexError, e:
                    log.error("Problem with error reporting ...")
                    log.error("Request Num: %i, Line: %s, dns: %i" %(request,line,len(dns) ))
            if ( len(line) > 5 ):
                log.warn(line.strip())
    return error_counter

def main(config, log):

    log.info("Starting Update Process")
    while 1:
        log.info("Starting Update")
        stats={}
        stats['update_start'] = time.time()

        new_ldif=""

        log.info("Reading static LDIF files ...")
        stats['read_start'] = time.time()
        ldif_files=os.listdir(config['BDII_LDIF_DIR'])
        for file_name in ldif_files:
            if ( file_name[-5:] == '.ldif' ):
                if ( not ((file_name[0] == '#') or (file_name[0] == '.'))):
                    file_url="file://%s/%s" %(config['BDII_LDIF_DIR'],file_name)
                    log.debug("Reading %s" % (file_url[7:]) )  
                    response = read_ldif(file_url)
                    new_ldif = new_ldif + response
                    
        stats['read_stop'] = time.time()

        log.info("Running Providers")
        stats['providers_start'] = time.time()
        providers=os.listdir(config['BDII_PROVIDER_DIR'])
        for provider in providers:
            if ( not ( provider[-1:] == '~' ) or (provider[0] == '#') or (provider[0] == '.')):
                log.debug("Running %s/%s" % (config['BDII_PROVIDER_DIR'],provider) )  
                response=read_ldif("%s/%s" % (config['BDII_PROVIDER_DIR'],provider))
                new_ldif = new_ldif + response                
                
        stats['providers_stop'] = time.time()

        new_dns = get_dns(new_ldif)
        ldif_modify=""

        log.info("Running Plugins")
        stats['plugins_start'] = time.time()
        plugins=os.listdir(config['BDII_PLUGIN_DIR'])
        for plugin in plugins:
            if ( not ( plugin[-1:] == '~' ) or (plugin[0] == '#') or (plugin[0] == '.')):
                log.debug("Running %s/%s" % (config['BDII_PLUGIN_DIR'],plugin) )  
                response = read_ldif("%s/%s" % (config['BDII_PLUGIN_DIR'],plugin))
                modify_dns = get_dns(response)
                for dn in modify_dns.keys():
                    if ( new_dns.has_key(dn)):
                        mod_entry = modify_entry( new_ldif[new_dns[dn][0]:new_dns[dn][1]],\
                                                  response[modify_dns[dn][0]:modify_dns[dn][1]])
                        start = len(new_ldif)
                        end = start + len(mod_entry)
                        new_dns[dn]=(start, end)
                        new_ldif = new_ldif + mod_entry
                    else:
                        ldif_modify += response[modify_dns[dn][0]:modify_dns[dn][1]]
        stats['plugins_stop'] = time.time()

        log.debug("Doing Fix")
        new_ldif = fix(new_dns, new_ldif)

        log.debug("Writing new_ldif to disk")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            dump_fh=open("%s/new.ldif" % (config['BDII_VAR_DIR']),'w')
            dump_fh.write(new_ldif)
            dump_fh.close()

        if ( not config['BDII_DAEMON'] ):
            print new_ldif
            sys.exit(0)
            
        log.info("Reading old LDIF file ...")
        stats['read_old_start'] = time.time()
        old_ldif_file = "%s/old.ldif" % (config['BDII_VAR_DIR'])
        if ( os.path.exists(old_ldif_file) ):
            old_ldif = read_ldif("file://%s" % (old_ldif_file))
        else:
            old_ldif = ""
        stats['read_old_stop'] = time.time()
        
        log.debug("Starting Diff")
        ldif_add=[]
        ldif_delete=[]

        new_dns = get_dns(new_ldif)
        old_dns = get_dns(old_ldif)

        for dn in new_dns.keys():
            if old_dns.has_key(dn):
                old = old_ldif[old_dns[dn][0]:old_dns[dn][1]].strip()
                new = new_ldif[new_dns[dn][0]:new_dns[dn][1]].strip()

                # If the entries are different we need to compare them
                if ( not new == old):
                    entry = ldif_diff(dn,old,new)
                    ldif_modify += entry
            else:
                ldif_add.append(dn)

        # Checking for removed entries
        for dn in old_dns.keys():
            if not new_dns.has_key(dn):
                ldif_delete.append(old_ldif[old_dns[dn][0] + 4:old_dns[dn][2]].strip())
        
        log.debug("Finished Diff")
        
        log.debug("Sorting Add Keys")
        ldif_add.sort(lambda x, y: cmp(len(x), len(y)))

        log.debug("Writing ldif_add to disk")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            dump_fh=open("%s/add.ldif" % (config['BDII_VAR_DIR']),'w')
            for dn in ldif_add:
                dump_fh.write(new_ldif[new_dns[dn][0]:new_dns[dn][1]])
                dump_fh.write("\n")
            dump_fh.close()

        log.debug("Adding New Entries")
        stats['db_update_start'] = time.time()

        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            error_file="%s/add.err" %(config['BDII_VAR_DIR'])
        else:
            error_file=tempfile.mktemp()

        roots = group_dns(ldif_add)
        suffixes = roots.keys()
        if "o=shadow" in suffixes:
            index = suffixes.index("o=shadow")
            if index > 0:
                suffixes[index] = suffixes[0]
                suffixes[0] = "o=shadow"

        add_error_counter = 0
        for root in suffixes:
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh=os.popen("ldapadd -d 256 -x -c -h %s -p %s -D %s -y %s >/dev/null 2>%s" %(config['BDII_HOSTNAME'], config['BDII_PORT'], bind, config['BDII_PASSWD_FILE'][bind], error_file), 'w')
                for dn in roots[root]:
                    input_fh.write(new_ldif[new_dns[dn][0]:new_dns[dn][1]])
                    input_fh.write("\n")
                input_fh.close()
            except IOError, KeyError:
                log.error("Could not add new entries to the database.")
              
            add_error_counter += log_errors(error_file,ldif_add)

            if ( not config['BDII_LOG_LEVEL'] == 'DEBUG' ):
                os.remove(error_file)
                        
        log.debug("Writing ldif_modify to disk")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            dump_fh=open("%s/modify.ldif" % (config['BDII_VAR_DIR']),'w')
            dump_fh.write(ldif_modify)
            dump_fh.close()

        log.debug("Modify New Entries")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            error_file="%s/modify.err" % (config['BDII_VAR_DIR'])
        else:
            error_file=tempfile.mktemp()

        ldif_modify_dns = get_dns(ldif_modify)
        roots = group_dns(ldif_modify_dns)
        modify_error_counter = 0
        for root in roots.keys():
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh=os.popen("ldapmodify -d 256 -x -c -h %s -p %s -D %s -y %s >/dev/null 2>%s" %(config['BDII_HOSTNAME'], config['BDII_PORT'], bind, config['BDII_PASSWD_FILE'][bind], error_file), 'w')
                for dn in roots[root]:
                    input_fh.write(ldif_modify[ldif_modify_dns[dn][0]:ldif_modify_dns[dn][1]])
                    input_fh.write("\n")
                input_fh.close()
            except IOError, KeyError:
                log.error("Could not modify entries in the database.")

            modify_error_counter += log_errors(error_file, ldif_modify_dns.keys())

            if ( not config['BDII_LOG_LEVEL'] == 'DEBUG' ):
                os.remove(error_file)

        log.debug("Sorting Delete Keys")
        ldif_delete.sort(lambda x, y: cmp(len(y), len(x)))

        log.debug("Writing ldif_delete to disk")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            dump_fh=open("%s/delete.ldif" % (config['BDII_VAR_DIR']),'w')
            for dn in ldif_delete:
                dump_fh.write("%s\n" % (dn))
            dump_fh.close()

        # Delayed delete Function
        if config['BDII_DELETE_DELAY'] > 0:
            log.debug("Doing Delayed Delete")
            delete_timestamp = time.time()

            # Get DNs of entries to be deleted not yet in delayed delete so their status can be updated
            new_delayed_delete_file = '%s/new_delayed_delete.pkl' % (config['BDII_VAR_DIR'])
            try:
                nfh = open(new_delayed_delete_file,'w')
                nfh.write("")
            except IOError:
                log.error("Unable to open new_delayed_delete file %s" % (new_delayed_delete))

            delayed_delete_file = '%s/delayed_delete.pkl' % (config['BDII_VAR_DIR'])
            if os.path.exists(delayed_delete_file):
                file_handle = open(delayed_delete_file, 'rb')
                delay_delete = pickle.load(file_handle)
                file_handle.close()
            else:
                delay_delete = {}
                
            # Add remove cache timestamps that have been readded
            for dn in delay_delete.keys():
                if dn not in ldif_delete:
                    log.debug("Removing %s from cache (readded)" %(dn,))
                    delay_delete.pop(dn)
                
            # Add current timestamp for new deletes
            for dn in ldif_delete:
                if dn not in delay_delete:
                    delay_delete[dn] = delete_timestamp
                    nfh.write("%s\n" % (dn))
            nfh.close()

            # Remove delayed deletes from LDIF or remove from cache
            for dn in delay_delete.keys():
                if delay_delete[dn] + config['BDII_DELETE_DELAY'] >= delete_timestamp:
                    ldif_delete.remove(dn)
                else:
                    delay_delete.pop(dn)
                    
            # Store Delayed Deletes
            log.debug("Storing delayed deletes")
            file_handle = open(delayed_delete_file, 'wb')
            pickle.dump(delay_delete, file_handle)
            file_handle.close()
                             
        log.debug("Deleting Old Entries")
        if ( config['BDII_LOG_LEVEL'] == 'DEBUG' ):
            error_file="%s/delete.err" % (config['BDII_VAR_DIR'])
        else:
            error_file=tempfile.mktemp()

            
        roots = group_dns(ldif_delete)
        delete_error_counter = 0
        for root in roots.keys():
            try:
                bind = root
                if "o=shadow" in config['BDII_PASSWD']:
                    if root == "o=grid":
                        bind = "o=shadow"
                input_fh=os.popen("ldapdelete -d 256 -x -c -h %s -p %s -D %s -y %s >/dev/null 2>%s" %(config['BDII_HOSTNAME'], config['BDII_PORT'], bind, config['BDII_PASSWD_FILE'][bind], error_file), 'w')
                for dn in roots[root]:
                    input_fh.write("%s\n" % (dn))
                    log.debug("Deleting %s" %(dn))
                input_fh.close()
            except IOError, KeyError:
                log.error("Could not delete old entries in the database.")

            delete_error_counter += log_errors(error_file, ldif_delete)

            if ( not config['BDII_LOG_LEVEL'] == 'DEBUG' ):
                os.remove(error_file)

        roots = group_dns(new_dns)
        stats['query_start'] = time.time()
        if ( os.path.exists("%s/old.ldif" % config['BDII_VAR_DIR']) ):
            os.remove("%s/old.ldif" % config['BDII_VAR_DIR'])
        if ( os.path.exists("%s/old.err" % config['BDII_VAR_DIR']) ):
            os.remove("%s/old.err" % config['BDII_VAR_DIR'])
        for root in roots.keys():
            # Stop flapping due to o=shadow
            if root == "o=shadow":
                command = "ldapsearch -LLL -x -h %s -p %s -b %s -s base >> %s/old.ldif 2>> %s/old.err" % (config['BDII_HOSTNAME'], config['BDII_PORT'], root, config['BDII_VAR_DIR'], config['BDII_VAR_DIR'])
            else:
                command = "ldapsearch -LLL -x -h %s -p %s -b %s >> %s/old.ldif 2>> %s/old.err" % (config['BDII_HOSTNAME'], config['BDII_PORT'], root, config['BDII_VAR_DIR'], config['BDII_VAR_DIR'])
            result = os.system(command)
            if ( result > 0):
                log.error("Query to self failed.")
        stats['query_stop'] = time.time()
        out_file="%s/archive/%s-snapshot.gz" % (config['BDII_VAR_DIR'], time.strftime('%y-%m-%d-%H-%M-%S'))
        log.debug("Creating GZIP file")
        os.system("gzip -c %s/old.ldif > %s" %(config['BDII_VAR_DIR'], out_file) )

        infosys_output=""
        if (len(old_ldif) == 0 ):
            log.debug("ldapadd o=infosys compression")
            command="ldapadd"
            
            infosys_output+="dn: o=infosys\n"
            infosys_output+="objectClass: organization\n"
            infosys_output+="o: infosys\n\n"
            infosys_output+="dn: CompressionType=zip,o=infosys\n"
            infosys_output+="objectClass: CompressedContent\n"
            infosys_output+="Hostname: %s\n" %(config['BDII_HOSTNAME'])
            infosys_output+="CompressionType: zip\n"
            infosys_output+="Data: file://%s\n\n" %(out_file)
        else:
            log.debug("ldapmodify o=infosys compression")
            command="ldapmodify"

            infosys_output+="dn: CompressionType=zip,o=infosys\n"
            infosys_output+="changetype: Modify\n"
            infosys_output+="replace: Data\n"
            infosys_output+="Data: file://%s\n\n" %(out_file)
        try:
            output_fh = os.popen("%s -x -c -h %s -p %s -D o=infosys -y %s >/dev/null" %(command, config['BDII_HOSTNAME'], config['BDII_PORT'], config['BDII_PASSWD_FILE']['o=infosys']), 'w')
            output_fh.write(infosys_output)
            output_fh.close()
        except IOError, KeyError:
            log.error("Could not add compressed data to the database.")

        old_files=os.popen("ls -t %s/archive" % (config['BDII_VAR_DIR']) ).readlines()
        log.debug("Deleting old GZIP files")
        for file in old_files[config['BDII_ARCHIVE_SIZE']:]:
            os.remove("%s/archive/%s" % (config['BDII_VAR_DIR'],file.strip()))

        stats['db_update_stop'] = time.time()
        stats['update_stop'] = time.time()

        stats['UpdateTime'] = int(stats['update_stop']
                                  - stats['update_start'])
        stats['ReadTime'] = int(stats['read_old_stop']
                                  - stats['read_old_start'])
        stats['ProvidersTime'] = int(stats['providers_stop']
                                     - stats['providers_start'])
        stats['PluginsTime'] = int(stats['plugins_stop']
                                     - stats['plugins_start'])
        stats['QueryTime'] = int(stats['query_stop']
                                   - stats['query_start'])
        stats['DBUpdateTime'] = int(stats['db_update_stop']
                                   - stats['db_update_start'])
        stats['TotalEntries'] = len(old_dns) 
        stats['NewEntries'] =  len(ldif_add)
        stats['ModifiedEntries'] = len(ldif_modify_dns.keys())
        stats['DeletedEntries'] =  len(ldif_delete)
        stats['FailedAdds'] =  add_error_counter
        stats['FailedModifies'] = modify_error_counter
        stats['FailedDeletes'] = delete_error_counter

        for key in stats.keys():
            if ( key.find("_") == -1 ):
                log.info("%s: %i" % (key, stats[key]) )

        infosys_output=""
        if (len(old_ldif) == 0 ):
            log.debug("ldapadd o=infosys updatestats")
            command="ldapadd"

            infosys_output+="dn: Hostname=%s,o=infosys\n" %(config['BDII_HOSTNAME'])
            infosys_output+="objectClass: UpdateStats\n"
            infosys_output+="Hostname: %s\n" %(config['BDII_HOSTNAME'])
            for key in stats.keys():
                if ( key.find("_") == -1):
                    infosys_output+="%s: %i\n" %(key, stats[key])
            infosys_output+="\n"
        else:
            log.debug("ldapmodify o=infosys updatestats")
            command="ldapmodify"

            infosys_output+="dn: Hostname=%s,o=infosys\n" %(config['BDII_HOSTNAME'])
            infosys_output+="changetype: Modify\n"
            for key in stats.keys():
                if ( key.find("_") == -1):
                    infosys_output+="replace: %s\n" %(key)
                    infosys_output+="%s: %i\n" %(key, stats[key])
                    infosys_output+="-\n"
            infosys_output+="\n"
        try:
            output_fh = os.popen("%s -x -c -h %s -p %s -D o=infosys -y %s >/dev/null" %(command, config['BDII_HOSTNAME'], config['BDII_PORT'], config['BDII_PASSWD_FILE']['o=infosys']), 'w')
            output_fh.write(infosys_output)
            output_fh.close()
        except IOError, KeyError:
            log.error("Could not add stats entries to the database.")

        old_ldif = None
        new_ldif = None
        new_dns = None
        ldif_delete = None
        ldif_add = None
        ldif_modify = None

        log.info("Sleeping for %i seconds" %(int(config['BDII_BREATHE_TIME'])))
        time.sleep(config['BDII_BREATHE_TIME'])

if __name__ == '__main__':

    config=parse_options()
    config=get_config(config)
    if ( config['BDII_DAEMON'] ):
        create_daemon(config['BDII_LOG_FILE'])
        # Giving some time for the init.d script to finish
        time.sleep(3)
    else:
        # connect stderr to log file
        e = os.open(config['BDII_LOG_FILE'], os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0644)
        os.dup2(e, 2)
        os.close(e)
        sys.stderr = os.fdopen(2, 'a', 0)
                        

    log=get_logger(config['BDII_LOG_FILE'],config['BDII_LOG_LEVEL'])
    main(config,log)


                
