Source code for arcresthelper.portalautomation

from __future__ import print_function
from __future__ import absolute_import

import gc
import sys, os, datetime
import json
import csv

from . import publishingtools
from . import orgtools
from . import common
from .securityhandlerhelper import securityhandlerhelper

try:
    import solutionreporttools
    from solutionreporttools import reporttools as ReportTools
    from solutionreporttools import dataprep as DataPrep
    reportToolsInstalled = True
except:
    reportToolsInstalled = False

#----------------------------------------------------------------------
[docs]def trace(): """Determines information about where an error was thrown. Returns: tuple: line number, filename, error message Examples: >>> try: ... 1/0 ... except: ... print("Error on '{}'\\nin file '{}'\\nwith error '{}'".format(*trace())) ... Error on 'line 1234' in file 'C:\\foo\\baz.py' with error 'ZeroDivisionError: integer division or modulo by zero' """ import traceback, inspect tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] filename = inspect.getfile(inspect.currentframe()) # script name + line number line = tbinfo.split(", ")[1] # Get Python syntax error # synerror = traceback.format_exc().splitlines()[-1] return line, filename, synerror
[docs]class portalautomation(securityhandlerhelper): _log = None
[docs] def setLog(self, log_file): """ References: :py:func:`arcresthelper.common.init_log` """ if log_file is None: log_file = "log.txt" self._log = common.init_log(log_file=log_file)
def __del__(self): if self._log is not None: common.close_log(log_file = self._log) #----------------------------------------------------------------------
[docs] def stageContent(self, configFiles, dateTimeFormat=None): """Parses a JSON configuration file to stage content. Args: configFiles (list): A list of JSON files on disk containing configuration data for staging content. dateTimeFormat (str): A valid date formatting directive, as understood by :py:meth:`datetime.datetime.strftime`. Defaults to ``None``, i.e., ``'%Y-%m-%d %H:%M'``. """ results = None groups = None items = None group = None content = None contentInfo = None startTime = None orgTools = None if dateTimeFormat is None: dateTimeFormat = '%Y-%m-%d %H:%M' scriptStartTime = datetime.datetime.now() try: print ("********************Stage Content Started********************") print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat)) if self.securityhandler.valid == False: print ("Login required") else: orgTools = orgtools.orgtools(securityinfo=self) if orgTools is None: print ("Error creating org tools") else: for configFile in configFiles: config = common.init_config_json(config_file=configFile) if config is not None: if 'ContentItems' in config: startTime = datetime.datetime.now() print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))) contentInfo = config['ContentItems'] for cont in contentInfo: content = cont['Content'] group = cont['ShareToGroup'] print ("Sharing content to: %s" % group) if os.path.isfile(content): with open(content, 'rb') as csvfile: items = [] groups = [] for row in csv.DictReader(csvfile,dialect='excel'): if cont['Type'] == "Group": groups.append(row['id']) elif cont['Type'] == "Items": items.append(row['id']) results = orgTools.shareItemsToGroup(shareToGroupName=group,items=items,groups=groups) print ("Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))) else: print ("Config file missing ContentItems section") else: print ("Config %s not found" % configFile) except(TypeError,ValueError,AttributeError) as e: print (e) except (common.ArcRestHelperError) as e: print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) except Exception as e: if (reportToolsInstalled): if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)): print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) finally: print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime)) print ("###############Stage Content Completed#################") print ("") #if orgTools is not None: #orgTools.dispose() results = None groups = None items = None group = None content = None contentInfo = None startTime = None orgTools = None del results del groups del items del group del content del contentInfo del startTime del orgTools gc.collect()
#----------------------------------------------------------------------
[docs] def createRoles(self, configFiles, dateTimeFormat=None): """Parses a JSON configuration file to create roles. Args: configFiles (list): A list of JSON files on disk containing configuration data for creating roles. dateTimeFormat (str): A valid date formatting directive, as understood by :py:meth:`datetime.datetime.strftime`. Defaults to ``None``, i.e., ``'%Y-%m-%d %H:%M'``. """ if dateTimeFormat is None: dateTimeFormat = '%Y-%m-%d %H:%M' scriptStartTime = datetime.datetime.now() try: print ("********************Create Roles********************") print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat)) if self.securityhandler.valid == False: print ("Login required") else: orgTools = orgtools.orgtools(securityinfo=self) if orgTools is None: print ("Error creating orgtools") else: for configFile in configFiles: config = common.init_config_json(config_file=configFile) if config is not None: startTime = datetime.datetime.now() print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))) roleInfos = config['Roles'] for roleInfo in roleInfos: createRoleResults = orgTools.createRole(roleInfo['Name'],roleInfo['Description'],roleInfo['Privileges']) else: print ("Config %s not found" % configFile) except(TypeError,ValueError,AttributeError) as e: print (e) except (common.ArcRestHelperError) as e: print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) except Exception as e: if (reportToolsInstalled): if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)): print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) finally: print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime)) print ("###############Create Groups Completed#################") print ("") #if orgTools is not None: #orgTools.dispose() groupInfo = None groupFile = None iconPath = None startTime = None thumbnail = None result = None config = None sciptPath = None orgTools = None del groupInfo del groupFile del iconPath del startTime del thumbnail del result del config del sciptPath del orgTools gc.collect()
#----------------------------------------------------------------------
[docs] def createGroups(self, configFiles, dateTimeFormat=None): """Parses a JSON configuration file to create groups. Args: configFiles (list): A list of JSON files on disk containing configuration data for creating groups. dateTimeFormat (str): A valid date formatting directive, as understood by :py:meth:`datetime.datetime.strftime`. Defaults to ``None``, i.e., ``'%Y-%m-%d %H:%M'``. """ groupInfo = None groupFile = None iconPath = None startTime = None thumbnail = None result = None config = None sciptPath = None orgTools = None if dateTimeFormat is None: dateTimeFormat = '%Y-%m-%d %H:%M' scriptStartTime = datetime.datetime.now() try: print ("********************Create Groups********************") print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat)) if self.securityhandler is None or \ self.securityhandler.valid == False: print ("Login required") else: orgTools = orgtools.orgtools(securityinfo=self) if orgTools is None: print ("Error creating orgtools") else: for configFile in configFiles: config = common.init_config_json(config_file=configFile) if config is not None: startTime = datetime.datetime.now() print ("Processing config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))) groupInfo = config['Groups'] groupFile = groupInfo['GroupInfo'] iconPath = groupInfo['IconPath'] if os.path.isfile(groupFile): with open(groupFile, 'rb') as csvfile: for row in csv.DictReader(csvfile,dialect='excel'): if os.path.isfile(os.path.join(iconPath,row['thumbnail'])): thumbnail = os.path.join(iconPath,row['thumbnail']) if not os.path.isabs(thumbnail): sciptPath = os.getcwd() thumbnail = os.path.join(sciptPath,thumbnail) result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \ isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly'],thumbnail=thumbnail) else: result = orgTools.createGroup(title=row['title'],description=row['description'],tags=row['tags'],snippet=row['snippet'],phone=row['phone'],access=row['access'],sortField=row['sortField'],sortOrder=row['sortOrder'], \ isViewOnly=row['isViewOnly'],isInvitationOnly=row['isInvitationOnly']) if result is None: pass else: print ("Group created: " + result.title) print ("Config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))) else: print ("Config %s not found" % configFile) except(TypeError,ValueError,AttributeError) as e: print (e) except (common.ArcRestHelperError) as e: print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) except Exception as e: if (reportToolsInstalled): if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)): print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) finally: print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime)) print ("###############Create Groups Completed#################") print ("") #if orgTools is not None: #orgTools.dispose() groupInfo = None groupFile = None iconPath = None startTime = None thumbnail = None result = None config = None sciptPath = None orgTools = None del groupInfo del groupFile del iconPath del startTime del thumbnail del result del config del sciptPath del orgTools gc.collect()
#----------------------------------------------------------------------
[docs] def publishfromconfig(self, configFiles, combinedApp=None, dateTimeFormat=None): """Parses a JSON configuration file to publish data. Args: configFiles (list): A list of JSON files on disk containing configuration data for publishing. combinedApp (str): A JSON file on disk containing configuration data for app publishing. Defaults to ``None``. dateTimeFormat (str): A valid date formatting directive, as understood by :py:meth:`datetime.datetime.strftime`. Defaults to ``None``, i.e., ``'%Y-%m-%d %H:%M'``. """ publishTools = None webmaps = None config = None resultsItems = None resultFS = None resultMaps = None resultApps = None combinedResults = None if dateTimeFormat is None: dateTimeFormat = '%Y-%m-%d %H:%M' scriptStartTime = datetime.datetime.now() try: webmaps = [] print ("********************Script Started********************") print ("Script started at %s" % scriptStartTime.strftime(dateTimeFormat)) # start report processing (moved out from under ArcREST logic. no AGO crednetials needed to run reports) for configFile in configFiles: config = common.init_config_json(config_file=configFile) if config is not None: if 'ReportDetails' in config: if reportToolsInstalled == False: print ("Report section is included in the config file but the solutionreporttools cannot be located") else: reportConfig = config['ReportDetails'] # This code checks to see if you want to export the data from SDE to a local GDB. The parameter is set in config file. # Could be performance gain to run locally. If you choose this option, both the report and the data prep in memory config # are modified so they can point to the local temp location. if 'RunReport' in reportConfig and (str(reportConfig['RunReport']).upper() =="TRUE" or str(reportConfig['RunReport']).upper() =="YES"): reportConfig = ReportTools.reportDataPrep(reportConfig) print ("-----Report Section Starting-----") startTime = datetime.datetime.now() print ("Processing reports in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))) ReportTools.create_report_layers_using_config(config=reportConfig) print ("Reports in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))) print ("-----Report Section Complete-----") if 'PublishingDetails' in config: publishingConfig = config['PublishingDetails'] if 'PublishData' in publishingConfig: publishData = publishingConfig['PublishData'] else: print ("PublishingDetails is missing the PublishData parameter: type string, values, True or False") publishData = 'TRUE' if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"): print (" ") print ("-----Publishing Section Starting-----") startTime = datetime.datetime.now() print ("Processing publishing in config %s, starting at: %s" % (configFile,startTime.strftime(dateTimeFormat))) publishTools = publishingtools.publishingtools(securityinfo=self) if publishTools.valid == False : print ("Error creating publishing tools: %s" % publishTools.message) else: print ("Publishing tools created: %s" % publishTools.message) resultFS = [] if 'Items' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Creating Items: %s" % str(startSectTime.strftime(dateTimeFormat))) resultsItems = publishTools.publishItems(items_info=publishingConfig['Items']) print ("Items created, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) if 'FeatureCollections' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Creating Feature Collection: %s" % str(startSectTime.strftime(dateTimeFormat))) resultFS = publishTools.publishFeatureCollections(configs=publishingConfig['FeatureCollections']) print ("Feature Collection published, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) if 'FeatureServices' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Creating Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat))) res = publishTools.publishFsFromMXD(fs_config=publishingConfig['FeatureServices']) if res is None: return resultFS = resultFS + res if len(resultFS) == 0: print ("Exiting, error creating feature services") return print ("Feature Services published, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) if 'ExistingServices' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Updating Existing Feature Services: %s" % str(startSectTime.strftime(dateTimeFormat))) resultES = publishTools.updateFeatureService(efs_config=publishingConfig['ExistingServices']) print ("Updating Existing Feature Services completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) if 'MapDetails' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Creating maps: %s" % str(startSectTime.strftime(dateTimeFormat))) resultMaps = publishTools.publishMap(maps_info=publishingConfig['MapDetails'],fsInfo=resultFS,itInfo=resultsItems) for maps in resultMaps: if 'MapInfo' in maps: if 'Results' in maps['MapInfo']: if 'itemId' in maps['MapInfo']['Results']: webmaps.append(maps['MapInfo']['Results']['itemId']) print ("Creating maps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) if 'AppDetails' in publishingConfig: startSectTime = datetime.datetime.now() print (" ") print ("Creating apps: %s" % str(startSectTime.strftime(dateTimeFormat))) resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=resultMaps,fsInfo=resultFS) print ("Creating apps completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) print ("Publishing complete in config %s completed, time to complete: %s" % (configFile, str(datetime.datetime.now() - startTime))) print ("-----Publishing Section Complete-----") else: print ("Config %s not found" % configFile) if combinedApp: if os.path.exists(combinedApp): print (" ") startSectTime = datetime.datetime.now() print ("Creating combined result: %s" % str(startSectTime.strftime(dateTimeFormat))) config = common.init_config_json(config_file=combinedApp) combinedResults = publishTools.publishCombinedWebMap(maps_info=config['PublishingDetails']['MapDetails'],webmaps=webmaps) if 'PublishingDetails' in config: publishingConfig = config['PublishingDetails'] if 'PublishData' in publishingConfig: publishData = publishingConfig['PublishData'] else: print ("PublishingDetails is missing the PublishData parameter: type string, values, True or False") publishData = 'TRUE' if (str(publishData).upper() =="TRUE" or str(publishData).upper() =="YES"): if 'AppDetails' in publishingConfig: resultApps = publishTools.publishApp(app_info=publishingConfig['AppDetails'],map_info=combinedResults) print ("Creating combind result completed, time to complete: %s" % str(datetime.datetime.now() - startSectTime)) except(TypeError,ValueError,AttributeError) as e: print (e) except (common.ArcRestHelperError) as e: print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) except Exception as e: if (reportToolsInstalled): if isinstance(e,(ReportTools.ReportToolsError,DataPrep.DataPrepError)): print ("error in function: %s" % e[0]['function']) print ("error on line: %s" % e[0]['line']) print ("error in file name: %s" % e[0]['filename']) print ("with error message: %s" % e[0]['synerror']) if 'arcpyError' in e[0]: print ("with arcpy message: %s" % e[0]['arcpyError']) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) else: line, filename, synerror = trace() print ("error on line: %s" % line) print ("error in file name: %s" % filename) print ("with error message: %s" % synerror) finally: print ("Script complete, time to complete: %s" % str(datetime.datetime.now() - scriptStartTime)) print ("###############Script Completed#################") print ("") if publishTools is not None: publishTools.dispose() publishTools = None webmaps = None config = None resultFS = None resultsItems = None resultMaps = None resultApps = None combinedResults = None del publishTools del webmaps del config del resultFS del resultMaps del resultApps del combinedResults del resultsItems gc.collect()