Skip to content

Commit

Permalink
calculate dataset extent on gpkg import
Browse files Browse the repository at this point in the history
  • Loading branch information
@enricofer
enricofer committed Jun 20, 2024
1 parent 73e7b06 commit 0960a73
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 63 deletions.
25 changes: 13 additions & 12 deletions webapp/djakart/admin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
pull_versione,
kart_cmd
)
SRID = os.environ.get("REPO_CRS")
SRID_CODE = SRID.split(":")[1]
#SRID = os.environ.get("REPO_CRS")
#SRID_CODE = SRID.split(":")[1]

class importForm(forms.Form):
nuovo_dataset = forms.FileField()
Expand Down Expand Up @@ -86,12 +86,13 @@ class versioniAdmin(DjangoObjectActions, admin.GISModelAdmin):#admin.OSMGeoAdmin
]
ordering = ['nome', ]

def change_view(self, *args, **kwargs):
#extra_context = kwargs["extra_context"] or {}
def change_view(self, request, object_id, **kwargs):
obj = version.objects.get(pk=object_id)
print
if not "extra_context" in kwargs:
kwargs["extra_context"] = {}
kwargs["extra_context"]['crs'] = SRID_CODE
return super(versioniAdmin, self).change_view(*args, **kwargs) #, extra_context=extra_context
kwargs["extra_context"]['crs'] = obj.crs.split(":")[1]
return super(versioniAdmin, self).change_view(request, object_id, **kwargs) #, extra_context=extra_context

def require_file(parameter,estensione):
def decorator(func):
Expand Down Expand Up @@ -180,7 +181,7 @@ def wrapper(modeladmin, request, obj, **kwargs):

context = {
"conflicts": json.dumps(conflicts),
"crs": SRID,
"crs": obj.crs,
"root_wms":os.environ.get("QGIS_SERVER_EXTERNAL","qgis_server_external") + '?MAP=/kart_versions/',
"base_name":obj.nome,
"version_name":version_name,
Expand Down Expand Up @@ -232,17 +233,17 @@ def get_readonly_fields(self, request, obj=None):
if not obj:
return ('clean','mapping_service_url', 'log', 'last_commit', 'status', 'mapa', 'get_project', 'merged')
if can_modify(request.user,obj):
return ('apply_map_extent', 'clean','mapping_service_url', 'log', 'last_commit', 'status', 'mapa', 'get_project', 'merged', 'nome', 'base')
return ('apply_map_extent', 'clean','mapping_service_url', 'crs','log', 'last_commit', 'status', 'mapa', 'get_project', 'merged', 'nome', 'base')
else:
return ('clean','mapping_service_url', "template_qgis", 'log', 'last_commit', 'status', 'mapa', 'get_project', 'merged', 'referente', 'riservato', 'nome', 'base', 'origine')
return ('clean','mapping_service_url', "template_qgis", 'crs', 'log', 'last_commit', 'status', 'mapa', 'get_project', 'merged', 'referente', 'riservato', 'nome', 'base', 'origine')

def get_fieldsets(self, request, obj=None):
if obj:
if obj.base:
return (
("intestazione", {
'classes': ('grp-collapse grp-open',),
'fields': ('nome', ('base','merged','clean',),'note','get_project','mapping_service_url',('referente','riservato'),'mapa',('extent','apply_map_extent'))
'fields': ('nome', ('base','merged','clean',),'note','get_project','mapping_service_url',('referente','riservato'),'mapa',('crs','extent','apply_map_extent'))
}),
("rapporti", {
'classes': ('grp-collapse grp-open',),
Expand All @@ -253,7 +254,7 @@ def get_fieldsets(self, request, obj=None):
return (
("intestazione", {
'classes': ('grp-collapse grp-open',),
'fields': ('nome', ('base','merged','clean',),'template_qgis','note','get_project','mapping_service_url',('referente','riservato'),'mapa',('extent','apply_map_extent'))
'fields': ('nome', ('base','merged','clean',),'template_qgis','note','get_project','mapping_service_url',('referente','riservato'),'mapa',('crs','extent','apply_map_extent'))
}),
("rapporti", {
'classes': ('grp-collapse grp-open',),
Expand All @@ -264,7 +265,7 @@ def get_fieldsets(self, request, obj=None):
return (
("intestazione", {
'classes': ('grp-collapse grp-open',),
'fields': ('nome', 'base','template_qgis','note','referente','riservato')
'fields': ('nome', 'base','template_qgis','crs','note','referente','riservato')
}),
)

Expand Down
110 changes: 70 additions & 40 deletions webapp/djakart/kart_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import json
import re
import requests
import sys

KART_EXE = "/opt/kart/kart_cli"

Expand All @@ -24,17 +25,17 @@
class KartException(Exception):
pass

def executeKart(commands, cmd=None ,path=None, jsonoutput=False, feedback=None):
def executeCmd(commands, cmd=False, path=None, jsonoutput=False, feedback=None):
if not cmd:
commands.insert(0, KART_EXE)
if jsonoutput:
commands.append("-ojson")

# The env PYTHONHOME from QGIS can interfere with Kart.
if not hasattr(executeKart, "env"):
executeKart.env = os.environ.copy()
if "PYTHONHOME" in executeKart.env:
executeKart.env.pop("PYTHONHOME")
if not hasattr(executeCmd, "env"):
executeCmd.env = os.environ.copy()
if "PYTHONHOME" in executeCmd.env:
executeCmd.env.pop("PYTHONHOME")

# always set the use helper env var as it is long lived and the setting may have changed
#executeKart.env['KART_USE_HELPER'] = '1' if setting(HELPERMODE) else ''
Expand All @@ -44,7 +45,7 @@ def executeKart(commands, cmd=None ,path=None, jsonoutput=False, feedback=None):
with subprocess.Popen(
commands,
#shell=os.name == "nt",
env=executeKart.env,
env=executeCmd.env,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.PIPE,
Expand Down Expand Up @@ -197,7 +198,7 @@ def crea_nuovo_repository(repo_name,bare=True,readonly_workingcopy=None):
if bare:
cmds = ["init", "--bare", repo_path]
if not os.path.exists(repo_path):
cmd = executeKart(cmds)
cmd = executeCmd(cmds)
return cmd


Expand All @@ -218,12 +219,12 @@ def crea_nuova_versione(nuova_versione,base,tipo="pg"):
uri = "" #local geopackage
if not os.path.exists(nuova_versione_path):
if not base:
init_cmd = executeKart(["init", nuova_versione_path])
init_cmd = executeCmd(["init", nuova_versione_path])
else:
#clone master
clone_cmd = executeKart(["clone", master_path, nuova_versione_path])
new_branch = executeKart(["--repo",nuova_versione_path,"checkout", "-b", nuova_versione])
new_wc = executeKart(["--repo",nuova_versione_path,"create-workingcopy", '--delete-existing', uri])
clone_cmd = executeCmd(["clone", master_path, nuova_versione_path])
new_branch = executeCmd(["--repo",nuova_versione_path,"checkout", "-b", nuova_versione])
new_wc = executeCmd(["--repo",nuova_versione_path,"create-workingcopy", '--delete-existing', uri])
grant_select_schema(nuova_versione)
#crea_fdw(nuova_versione)
serial_pk_setup(nuova_versione)
Expand All @@ -234,28 +235,28 @@ def merge_versione(versione, abort=False, confirm=False):
#clone master
#commit_versione(versione, '"merge %s"' % versione)
if abort:
abort_cmd = executeKart(["--repo", versione_path, "merge", "--abort"])
abort_cmd = executeCmd(["--repo", versione_path, "merge", "--abort"])
elif confirm:
status = json.loads(status_versione(versione, as_json=True))
theirs = status["kart.status/v2"]["merging"]["theirs"]
versione_merging = theirs.get("branch") or theirs.get("abbrevCommit")
confirm_cmd = executeKart(["--repo", versione_path, "merge", "--continue","-m","merge %s con risoluzione conflitti" % versione_merging])
confirm_cmd = executeCmd(["--repo", versione_path, "merge", "--continue","-m","merge %s con risoluzione conflitti" % versione_merging])
else:
master_path = get_remote(versione)
master_versione = os.path.split(master_path)[-1]
push_cmd = executeKart(["--repo", versione_path, "push", "origin", versione])
push_cmd = executeCmd(["--repo", versione_path, "push", "origin", versione])
#chkout_cmd = executeKart(["--repo", master_path, "checkout", "main"])
merge_cmd = executeKart(["--repo",master_path,"merge","-m","merge "+versione, versione])
merge_cmd = executeCmd(["--repo",master_path,"merge","-m","merge "+versione, versione])
if ("Nothing to commit, working copy clean" in status_versione(master_path)) or ("No working copy" in status_versione(master_path)):
#non dovrebbe cancellare branch con conflitti in fase di merge
delete_branch_cmd = executeKart(["--repo",master_path,"branch","-d",versione])
delete_branch_cmd = executeCmd(["--repo",master_path,"branch","-d",versione])
grant_select_schema(master_versione)
grant_select_schema(versione)

def clone_versione(versione, target):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
clone_cmd = executeKart(["clone", versione_path, target])
clone_cmd = executeCmd(["clone", versione_path, target])

def config_user_versione(versione, username, useremail):
versione_path = os.path.join(settings.KART_REPO,versione)
Expand All @@ -281,34 +282,63 @@ def pull_versione(versione):
versione_path = os.path.join(settings.KART_REPO,versione)
#master_path = get_remote(versione)
if os.path.exists(versione_path):
push_cmd = executeKart(["--repo", versione_path, "pull", "origin"])
push_cmd = executeCmd(["--repo", versione_path, "pull", "origin"])

def kart_cmd(versione,args):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
kart_cmd = executeKart(["--repo", versione_path] + args)
kart_cmd = executeCmd(["--repo", versione_path] + args)
return kart_cmd

def importa_dataset(versione,ds_path):
def importa_dataset(versione,ds_path,max_extent=None):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
importa_cmd = executeKart(["--repo", versione_path, "import", "--replace-existing", "-a", ds_path])
if not max_extent:
max_extent = [sys.float_info.max, sys.float_info.max, -sys.float_info.max, -sys.float_info.max]
#EXTRACT EXTENT
ogrinfo_pass1 = executeCmd(["/usr/bin/ogrinfo", ds_path],cmd=True)
#^[0-9]+: .*$
print("ogrinfo_pass1",ogrinfo_pass1)
lyrs = []
for group in re.findall(r"^[0-9]+: .*(?=\()", ogrinfo_pass1, flags=re.MULTILINE):
print (group)
lyrs.append(group)
clean_group = group.split(":")[-1].strip()
ogrinfo_pass2 = executeCmd(["/usr/bin/ogrinfo", "-so", ds_path, clean_group],cmd=True)
extent_row = re.findall(r"^Extent.*$",ogrinfo_pass2, flags=re.MULTILINE)
if extent_row:
extent_json = extent_row[0].replace("Extent: (","[").replace(") - (",", ").replace(")","]")
extent = json.loads(extent_json)
print ("group %s %s" % (group, extent))
if extent[0] < max_extent[0]:
max_extent[0] = extent[0]
if extent[1] < max_extent[1]:
max_extent[1] = extent[1]
if extent[2] > max_extent[2]:
max_extent[2] = extent[2]
if extent[3] > max_extent[3]:
max_extent[3] = extent[3]

print ("MAXEXTENT", max_extent)

importa_cmd = executeCmd(["--repo", versione_path, "import", "--replace-existing", "-a", ds_path])
grant_select_schema(versione)
return importa_cmd

return max_extent

def commit_versione(versione, messaggio):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
#recover_uncommitted_nulls(versione) #necessario per evitare successive exceptions di kart
commit_cmd = executeKart(["--repo", versione_path, "commit", "-m", messaggio])
commit_cmd = executeCmd(["--repo", versione_path, "commit", "-m", messaggio])

def elimina_versione(canc_versione):
canc_versione_path = os.path.join(settings.KART_REPO,canc_versione)
pgschema = elimina_pg_schema(canc_versione)
if os.path.exists(canc_versione_path):
#clone master
if pgschema:
rm_cmd = executeKart(["rm","-Rf",canc_versione_path],cmd=True)
rm_cmd = executeCmd(["rm","-Rf",canc_versione_path],cmd=True)
else:
print ("ERRORE elimina pg_uri")

Expand All @@ -319,15 +349,15 @@ def undo_commit_versione(versione, force=None):
jlog = log_versione(versione,jsonoutput=True)
hash = jlog[1]["commit"]
if force:
cmd = executeKart(["--repo",versione_path,"reset","--force",hash])
cmd = executeCmd(["--repo",versione_path,"reset","--force",hash])
else:
cmd = executeKart(["--repo",versione_path,"reset",hash])
cmd = executeCmd(["--repo",versione_path,"reset",hash])
return cmd

def restore_versione(versione):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"restore"])
cmd = executeCmd(["--repo",versione_path,"restore"])
return cmd

def status_versione(versione, as_json=False):
Expand All @@ -337,7 +367,7 @@ def status_versione(versione, as_json=False):
params = ["--repo",versione_path,"status"]
if as_json:
params = params + ["-o", "json"]
cmd = executeKart(params)
cmd = executeCmd(params)
return cmd
except KartException as E:
return str(E)
Expand All @@ -347,15 +377,15 @@ def status_versione(versione, as_json=False):
def show_versione(versione, jsonoutput=False):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"show"], jsonoutput=jsonoutput)
cmd = executeCmd(["--repo",versione_path,"show"], jsonoutput=jsonoutput)
merged_list = cmd.replace("* ","").replace(" ","").split("\n")
return merged_list

def merged_list_versione(versione):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
try:
cmd = executeKart(["--repo",versione_path,"branch","--merged"], jsonoutput=False)
cmd = executeCmd(["--repo",versione_path,"branch","--merged"], jsonoutput=False)
return cmd
except Exception as E:
print (E)
Expand All @@ -364,7 +394,7 @@ def merged_list_versione(versione):
def log_versione(versione, jsonoutput=False):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"log"], jsonoutput=jsonoutput)
cmd = executeCmd(["--repo",versione_path,"log"], jsonoutput=jsonoutput)
return cmd


Expand All @@ -382,18 +412,18 @@ def genera_diff_versione(versione, hash=None, prev=None, format='html'):
with open("/tmp/diff-view.html", "w", encoding="utf8") as diff_template:
diff_template.write(response.text)
if hash == 'HEAD':
cmd = executeKart(["--repo",versione_path,"diff","-o","html", "--html-template", '/tmp/diff-view.html', "--crs", SRID, "--output", "-", hash])
cmd = executeCmd(["--repo",versione_path,"diff","-o","html", "--html-template", '/tmp/diff-view.html', "--crs", SRID, "--output", "-", hash])
else:
cmd = executeKart(["--repo",versione_path,"diff","-o","html", "--html-template", '/tmp/diff-view.html', "--crs", SRID, "--output", "-", prev, hash])
cmd = executeCmd(["--repo",versione_path,"diff","-o","html", "--html-template", '/tmp/diff-view.html', "--crs", SRID, "--output", "-", prev, hash])
elif format == 'json':
cmd = executeKart(["--repo",versione_path,"diff","-o","json", "--output", "-", hash])
cmd = executeCmd(["--repo",versione_path,"diff","-o","json", "--output", "-", hash])
return cmd

def conflitti_versione(versione):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
conflicts_dir = os.path.join(versione_path,'conflicts')
cmd = executeKart(["--repo",versione_path,"conflicts", "--crs", SRID,"-o","geojson","--output",conflicts_dir])
cmd = executeCmd(["--repo",versione_path,"conflicts", "--crs", SRID,"-o","geojson","--output",conflicts_dir])
feats = []
for gj in os.listdir(conflicts_dir):
conflict_path = os.path.join(conflicts_dir,gj)
Expand All @@ -409,13 +439,13 @@ def conflitti_versione(versione):
def resolve_conflitto(versione, tag_conflitto, risoluzione):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"resolve","--with",risoluzione,tag_conflitto])
cmd = executeCmd(["--repo",versione_path,"resolve","--with",risoluzione,tag_conflitto])
return cmd

def get_remote(versione, remote="origin", method='push'):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"remote","-v"])
cmd = executeCmd(["--repo",versione_path,"remote","-v"])
if cmd:
q = "(?<={})([^\n]*)(?=\({}\))".format(remote,method)
test = re.search(q, cmd)
Expand All @@ -424,7 +454,7 @@ def get_remote(versione, remote="origin", method='push'):
def list_versioned_tables(versione):
versione_path = os.path.join(settings.KART_REPO,versione.replace("_pub",""))
if os.path.exists(versione_path):
cmd = executeKart(["--repo",versione_path,"data","ls"])
cmd = executeCmd(["--repo",versione_path,"data","ls"])
ds_list = cmd.split("\n")
return [ds for ds in ds_list if ds != ""]

Expand All @@ -438,7 +468,7 @@ def geo_tables(versione):
if os.path.exists(versione_path):
geom_tables = []
for tab in list_versioned_tables(versione):
cmd = executeKart(["--repo",versione_path,"meta","get",tab])
cmd = executeCmd(["--repo",versione_path,"meta","get",tab])
capture = re.search("""(?<=(schema.json\n))(\w|\d|\n|[().,\-:;@#$%^&*\[\]"'+–®°⁰!?{}|`~]| )+?(?=(\]))""", cmd)
if capture:
tab_meta = json.loads(capture[0]+"]")
Expand All @@ -451,7 +481,7 @@ def get_metadata(versione,tab):
versione_path = os.path.join(settings.KART_REPO,versione)
if os.path.exists(versione_path):
try:
cmd = executeKart(["--repo",versione_path,"meta","get",tab,"-o","json"])
cmd = executeCmd(["--repo",versione_path,"meta","get",tab,"-o","json"])
return json.loads(cmd)
except KartException as E:
return {"error":True,"result":str(E)}
Expand Down
Loading

0 comments on commit 0960a73

Please sign in to comment.