Source code for arcresthelper.featureservicetools
from __future__ import print_function
from __future__ import absolute_import
from .securityhandlerhelper import securityhandlerhelper
dateTimeFormat = '%Y-%m-%d %H:%M'
import arcrest
from arcrest.agol import FeatureLayer
from arcrest.agol import FeatureService
from arcrest.ags import FeatureService
from arcrest.hostedservice import AdminFeatureService
from arcrest.common.spatial import scratchFolder, scratchGDB, json_to_featureclass
from arcrest.common.general import FeatureSet
from arcresthelper.common import chunklist
import datetime, time
import json
import os
from . import common
import gc
try:
import arcpy
arcpyFound = True
except:
arcpyFound = False
import traceback, inspect, sys
import collections
#----------------------------------------------------------------------
[docs]def trace():
"""Determines information about where an error was thrown.
Returns:
tuple: line number, filename, error message
Examples:
>>> try:
... 1/0
... except:
... print("Error on '{}'\\nin file '{}'\\nwith error '{}'".format(*trace()))
...
Error on 'line 1234'
in file 'C:\\foo\\baz.py'
with error 'ZeroDivisionError: integer division or modulo by zero'
"""
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
filename = inspect.getfile(inspect.currentframe())
# script name + line number
line = tbinfo.split(", ")[1]
# Get Python syntax error
#
synerror = traceback.format_exc().splitlines()[-1]
return line, filename, synerror
[docs]class featureservicetools(securityhandlerhelper):
#----------------------------------------------------------------------
[docs] def RemoveAndAddFeatures(self, url, pathToFeatureClass, id_field, chunksize=1000):
"""Deletes all features in a feature service and uploads features from a feature class on disk.
Args:
url (str): The URL of the feature service.
pathToFeatureClass (str): The path of the feature class on disk.
id_field (str): The name of the field in the feature class to use for chunking.
chunksize (int): The maximum amount of features to upload at a time. Defaults to 1000.
Raises:
ArcRestHelperError: if ``arcpy`` can't be found.
"""
fl = None
try:
if arcpyFound == False:
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "ArcPy required for this function"
})
arcpy.env.overwriteOutput = True
tempaddlayer= 'ewtdwedfew'
if not arcpy.Exists(pathToFeatureClass):
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "%s does not exist" % pathToFeatureClass
}
)
fields = arcpy.ListFields(pathToFeatureClass,wild_card=id_field)
if len(fields) == 0:
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "%s field does not exist" % id_field
})
strFld = True
if fields[0].type != 'String':
strFld = False
fl = FeatureLayer(
url=url,
securityHandler=self._securityHandler)
id_field_local = arcpy.AddFieldDelimiters(pathToFeatureClass, id_field)
idlist = []
print( arcpy.GetCount_management(in_rows=pathToFeatureClass).getOutput(0) + " features in the layer")
with arcpy.da.SearchCursor(pathToFeatureClass, (id_field)) as cursor:
allidlist = []
for row in cursor:
if (strFld):
idlist.append("'" + row[0] +"'")
else:
idlist.append(row[0])
if len(idlist) >= chunksize:
allidlist.append(idlist)
idlist = []
if len(idlist) > 0:
allidlist.append(idlist)
for idlist in allidlist:
idstring = ' in (' + ','.join(idlist) + ')'
sql = id_field + idstring
sqlLocalFC = id_field_local + idstring
results = fl.deleteFeatures(where=sql,
rollbackOnFailure=True)
if 'error' in results:
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror":results['error']
})
elif 'deleteResults' in results:
print ("%s features deleted" % len(results['deleteResults']))
for itm in results['deleteResults']:
if itm['success'] != True:
print (itm)
else:
print (results)
arcpy.MakeFeatureLayer_management(pathToFeatureClass,tempaddlayer,sqlLocalFC)
results = fl.addFeatures(fc=tempaddlayer)
if 'error' in results:
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror":results['error']
})
elif 'addResults' in results:
print ("%s features added" % len(results['addResults']))
for itm in results['addResults']:
if itm['success'] != True:
print (itm)
else:
print (results)
idlist = []
if 'error' in results:
raise common.ArcRestHelperError({
"function": "RemoveAndAddFeatures",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror":results['error']
})
else:
print (results)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "create_report_layers_using_config",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
}
)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
gc.collect()
#----------------------------------------------------------------------
[docs] def EnableEditingOnService(self, url, definition = None):
"""Enables editing capabilities on a feature service.
Args:
url (str): The URL of the feature service.
definition (dict): A dictionary containing valid definition values. Defaults to ``None``.
Returns:
dict: The existing feature service definition capabilities.
When ``definition`` is not provided (``None``), the following values are used by default:
+------------------------------+------------------------------------------+
| Key | Value |
+------------------------------+------------------------------------------+
| hasStaticData | ``False`` |
+------------------------------+------------------------------------------+
| allowGeometryUpdates | ``True`` |
+------------------------------+------------------------------------------+
| enableEditorTracking | ``False`` |
+------------------------------+------------------------------------------+
| enableOwnershipAccessControl | ``False`` |
+------------------------------+------------------------------------------+
| allowOthersToUpdate | ``True`` |
+------------------------------+------------------------------------------+
| allowOthersToDelete | ``True`` |
+------------------------------+------------------------------------------+
| capabilities | ``"Query,Editing,Create,Update,Delete"`` |
+------------------------------+------------------------------------------+
"""
adminFS = AdminFeatureService(url=url, securityHandler=self._securityHandler)
if definition is None:
definition = collections.OrderedDict()
definition['hasStaticData'] = False
definition['allowGeometryUpdates'] = True
definition['editorTrackingInfo'] = {}
definition['editorTrackingInfo']['enableEditorTracking'] = False
definition['editorTrackingInfo']['enableOwnershipAccessControl'] = False
definition['editorTrackingInfo']['allowOthersToUpdate'] = True
definition['editorTrackingInfo']['allowOthersToDelete'] = True
definition['capabilities'] = "Query,Editing,Create,Update,Delete"
existingDef = {}
existingDef['capabilities'] = adminFS.capabilities
existingDef['allowGeometryUpdates'] = adminFS.allowGeometryUpdates
enableResults = adminFS.updateDefinition(json_dict=definition)
if 'error' in enableResults:
return enableResults['error']
adminFS = None
del adminFS
print (enableResults)
return existingDef
#----------------------------------------------------------------------
[docs] def enableSync(self, url, definition = None):
"""Enables Sync capability for an AGOL feature service.
Args:
url (str): The URL of the feature service.
definition (dict): A dictionary containing valid definition values. Defaults to ``None``.
Returns:
dict: The result from :py:func:`arcrest.hostedservice.service.AdminFeatureService.updateDefinition`.
"""
adminFS = AdminFeatureService(url=url, securityHandler=self._securityHandler)
cap = str(adminFS.capabilities)
existingDef = {}
enableResults = 'skipped'
if 'Sync' in cap:
return "Sync is already enabled"
else:
capItems = cap.split(',')
capItems.append('Sync')
existingDef['capabilities'] = ','.join(capItems)
enableResults = adminFS.updateDefinition(json_dict=existingDef)
if 'error' in enableResults:
return enableResults['error']
adminFS = None
del adminFS
return enableResults
#----------------------------------------------------------------------
[docs] def disableSync(self, url, definition = None):
"""Disables Sync capabilities for an AGOL feature service.
Args:
url (str): The URL of the feature service.
definition (dict): A dictionary containing valid definition values. Defaults to ``None``.
Returns:
dict: The result from :py:func:`arcrest.hostedservice.service.AdminFeatureService.updateDefinition`.
"""
adminFS = AdminFeatureService(url=url, securityHandler=self._securityHandler)
cap = str(adminFS.capabilities)
existingDef = {}
enableResults = 'skipped'
if 'Sync' in cap:
capItems = cap.split(',')
if 'Sync' in capItems:
capItems.remove('Sync')
existingDef['capabilities'] = ','.join(capItems)
enableResults = adminFS.updateDefinition(json_dict=existingDef)
if 'error' in enableResults:
return enableResults['error']
adminFS = None
del adminFS
return enableResults
#----------------------------------------------------------------------
[docs] def GetFeatureService(self, itemId, returnURLOnly=False):
"""Obtains a feature service by item ID.
Args:
itemId (int): The feature service's item ID.
returnURLOnly (bool): A boolean value to return the URL of the feature service. Defaults to ``False``.
Returns:
When ``returnURLOnly`` is ``True``, the URL of the feature service is returned.
When ``False``, the result from :py:func:`arcrest.agol.services.FeatureService` or :py:func:`arcrest.ags.services.FeatureService`.
"""
admin = None
item = None
try:
admin = arcrest.manageorg.Administration(securityHandler=self._securityHandler)
if self._securityHandler.valid == False:
self._valid = self._securityHandler.valid
self._message = self._securityHandler.message
return None
item = admin.content.getItem(itemId=itemId)
if item.type == "Feature Service":
if returnURLOnly:
return item.url
else:
fs = arcrest.agol.FeatureService(
url=item.url,
securityHandler=self._securityHandler)
if fs.layers is None or len(fs.layers) == 0 :
fs = arcrest.ags.FeatureService(
url=item.url)
return fs
return None
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "GetFeatureService",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
admin = None
item = None
del item
del admin
gc.collect()
#----------------------------------------------------------------------
[docs] def GetLayerFromFeatureServiceByURL(self, url, layerName="", returnURLOnly=False):
"""Obtains a layer from a feature service by URL reference.
Args:
url (str): The URL of the feature service.
layerName (str): The name of the layer. Defaults to ``""``.
returnURLOnly (bool): A boolean value to return the URL of the layer. Defaults to ``False``.
Returns:
When ``returnURLOnly`` is ``True``, the URL of the layer is returned.
When ``False``, the result from :py:func:`arcrest.agol.services.FeatureService` or :py:func:`arcrest.ags.services.FeatureService`.
"""
fs = None
try:
fs = arcrest.agol.FeatureService(
url=url,
securityHandler=self._securityHandler)
return self.GetLayerFromFeatureService(fs=fs,layerName=layerName,returnURLOnly=returnURLOnly)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "GetLayerFromFeatureServiceByURL",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fs = None
del fs
gc.collect()
#----------------------------------------------------------------------
[docs] def GetLayerFromFeatureService(self, fs, layerName="", returnURLOnly=False):
"""Obtains a layer from a feature service by feature service reference.
Args:
fs (FeatureService): The feature service from which to obtain the layer.
layerName (str): The name of the layer. Defaults to ``""``.
returnURLOnly (bool): A boolean value to return the URL of the layer. Defaults to ``False``.
Returns:
When ``returnURLOnly`` is ``True``, the URL of the layer is returned.
When ``False``, the result from :py:func:`arcrest.agol.services.FeatureService` or :py:func:`arcrest.ags.services.FeatureService`.
"""
layers = None
table = None
layer = None
sublayer = None
try:
layers = fs.layers
if (layers is None or len(layers) == 0) and fs.url is not None:
fs = arcrest.ags.FeatureService(
url=fs.url)
layers = fs.layers
if layers is not None:
for layer in layers:
if layer.name == layerName:
if returnURLOnly:
return fs.url + '/' + str(layer.id)
else:
return layer
elif not layer.subLayers is None:
for sublayer in layer.subLayers:
if sublayer == layerName:
return sublayer
if fs.tables is not None:
for table in fs.tables:
if table.name == layerName:
if returnURLOnly:
return fs.url + '/' + str(layer.id)
else:
return table
return None
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "GetLayerFromFeatureService",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
layers = None
table = None
layer = None
sublayer = None
del layers
del table
del layer
del sublayer
gc.collect()
#----------------------------------------------------------------------
[docs] def AddFeaturesToFeatureLayer(self, url, pathToFeatureClass, chunksize=0, lowerCaseFieldNames=False):
"""Appends local features to a hosted feature service layer.
Args:
url (str): The URL of the feature service layer.
pathToFeatureClass (str): The path of the feature class on disk.
chunksize (int): The maximum amount of features to upload at a time. Defaults to 0.
lowerCaseFieldNames (bool): A boolean value indicating if field names should be converted
to lowercase before uploading. Defaults to ``False``.
Returns:
The result from :py:func:`arcrest.agol.services.FeatureLayer.addFeatures`.
Raises:
ArcRestHelperError: if ``arcpy`` can't be found.
Notes:
If publishing to a PostgreSQL database, it is suggested to to set ``lowerCaseFieldNames`` to ``True``.
"""
if arcpyFound == False:
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": inspect.currentframe().f_back.f_lineno,
"filename": 'featureservicetools',
"synerror": "ArcPy required for this function"
})
fl = None
try:
fl = FeatureLayer(
url=url,
securityHandler=self._securityHandler)
if chunksize > 0:
fc = os.path.basename(pathToFeatureClass)
inDesc = arcpy.Describe(pathToFeatureClass)
oidName = arcpy.AddFieldDelimiters(pathToFeatureClass,inDesc.oidFieldName)
arr = arcpy.da.FeatureClassToNumPyArray(pathToFeatureClass, (oidName))
syncSoFar = 0
messages = {'addResults':[],'errors':[]}
total = len(arr)
errorCount = 0
if total == '0':
print ("0 features in %s" % pathToFeatureClass)
return "0 features in %s" % pathToFeatureClass
print ("%s features in layer" % (total))
arcpy.env.overwriteOutput = True
if int(total) < int(chunksize):
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
else:
newArr = chunklist(arr,chunksize)
exprList = ["{0} >= {1} AND {0} <= {2}".format(oidName, nArr[0][0], nArr[len(nArr)-1][0])
for nArr in newArr]
for expr in exprList:
UploadLayer = arcpy.MakeFeatureLayer_management(pathToFeatureClass, 'TEMPCOPY', expr).getOutput(0)
#print(arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0) + " features in the chunk")
results = fl.addFeatures(fc=UploadLayer,lowerCaseFieldNames=lowerCaseFieldNames)
chunkCount = arcpy.GetCount_management(in_rows=UploadLayer).getOutput(0)
print(chunkCount + " features in the chunk")
if chunkCount > 0:
if results is not None and 'addResults' in results and results['addResults'] is not None:
featSucces = 0
for result in results['addResults']:
if 'success' in result:
if result['success'] == False:
if 'error' in result:
errorCount = errorCount + 1
print ("\tError info: %s" % (result))
else:
featSucces = featSucces + 1
syncSoFar = syncSoFar + featSucces
print ("%s features added in this chunk" % (featSucces))
print ("%s/%s features added, %s errors" % (syncSoFar,total,errorCount ))
if 'addResults' in messages:
messages['addResults'] = messages['addResults'] + results['addResults']
else:
messages['addResults'] = results['addResults']
else:
messages['errors'] = result
return messages
else:
return fl.addFeatures(fc=pathToFeatureClass,lowerCaseFieldNames=lowerCaseFieldNames)
except arcpy.ExecuteError:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
"arcpyError": arcpy.GetMessages(2),
}
)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "AddFeaturesToFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
#----------------------------------------------------------------------
[docs] def DeleteFeaturesFromFeatureLayer(self, url, sql, chunksize=0):
"""Removes features from a hosted feature service layer by SQL query.
Args:
url (str): The URL of the feature service layer.
sql (str): The SQL query to apply against the feature service.
Those features that satisfy the query will be deleted.
chunksize (int): The maximum amount of features to remove at a time. Defaults to 0.
Returns:
The result from :py:func:`arcrest.agol.services.FeatureLayer.deleteFeatures`.
Notes:
If you want to delete all features, it is suggested to use the SQL query ``"1=1"``.
"""
fl = None
try:
fl = FeatureLayer(
url=url,
securityHandler=self._securityHandler)
totalDeleted = 0
if chunksize > 0:
qRes = fl.query(where=sql, returnIDsOnly=True)
if 'error' in qRes:
print (qRes)
return qRes
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return {'success':True,'message': "No features matched the query"}
i = 0
print ("%s features to be deleted" % total)
while(i <= len(oids)):
oidsDelete = ','.join(str(e) for e in oids[i:i+chunksize])
if oidsDelete == '':
continue
else:
results = fl.deleteFeatures(objectIds=oidsDelete)
if 'deleteResults' in results:
totalDeleted += len(results['deleteResults'])
print ("%s%% Completed: %s/%s " % (int(totalDeleted / float(total) *100), totalDeleted, total))
i += chunksize
else:
print (results)
return {'success':True,'message': "%s deleted" % totalDeleted}
qRes = fl.query(where=sql, returnIDsOnly=True)
if 'objectIds' in qRes:
oids = qRes['objectIds']
if len(oids)> 0 :
print ("%s features to be deleted" % len(oids))
results = fl.deleteFeatures(where=sql)
if 'deleteResults' in results:
totalDeleted += len(results['deleteResults'])
return {'success':True,'message': "%s deleted" % totalDeleted}
else:
return results
return {'success':True,'message': "%s deleted" % totalDeleted}
else:
print (qRes)
else:
results = fl.deleteFeatures(where=sql)
if results is not None:
if 'deleteResults' in results:
return {'success':True,'message': totalDeleted + len(results['deleteResults'])}
else:
return results
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "DeleteFeaturesFromFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()
#----------------------------------------------------------------------
[docs] def QueryAllFeatures(self, url, sql, out_fields="*", chunksize=1000, savePath=None):
"""Performs an SQL query against a hosted feature service layer.
Args:
url (str): The URL of the feature service layer.
sql (str): The SQL query to apply against the feature service.
Those features that satisfy the query will be returned.
out_fields (str): A comma delimited list of field names to return.
Defaults to ``"*"``, i.e., return all fields
chunksize (int): The maximum amount of features to query at a time. Defaults to 1000.
savePath (str): The full path on disk where the features will be saved. Defaults to ``None``.
Returns:
When ``savePath`` is not provided (``None``), the result from
:py:func:`arcrest.agol.services.FeatureLayer.query`.
When ``savePath`` is provided, the result from
:py:func:`arcrest.common.general.FeatureSet.save`.
"""
fl = None
try:
fl = FeatureLayer(url=url, securityHandler=self._securityHandler)
qRes = fl.query(where=sql, returnIDsOnly=True)
if 'error' in qRes:
print (qRes)
return qRes
elif 'objectIds' in qRes:
oids = qRes['objectIds']
total = len(oids)
if total == 0:
return {'success':True, 'message':"No features matched the query"}
print ("%s features to be downloaded" % total)
chunksize = min(chunksize, fl.maxRecordCount)
combinedResults = None
totalQueried = 0
for chunk in chunklist(l=oids, n=chunksize):
oidsQuery = ",".join(map(str, chunk))
if not oidsQuery:
continue
else:
results = fl.query(objectIds=oidsQuery,
returnGeometry=True,
out_fields=out_fields)
if isinstance(results,FeatureSet):
if combinedResults is None:
combinedResults = results
else:
for feature in results.features:
combinedResults.features.append(feature)
totalQueried += len(results.features)
print("{:.0%} Completed: {}/{}".format(totalQueried / float(total), totalQueried, total))
else:
print (results)
if savePath is None or savePath == '':
return combinedResults
else:
return combinedResults.save(*os.path.split(savePath))
else:
print (qRes)
except:
line, filename, synerror = trace()
raise common.ArcRestHelperError({
"function": "QueryFeatureLayer",
"line": line,
"filename": filename,
"synerror": synerror,
}
)
finally:
fl = None
del fl
gc.collect()