Skip to content
Snippets Groups Projects
Commit 52f4f5d7 authored by Jan Beutel's avatar Jan Beutel
Browse files

added switches for image resizing, image size selection

parent 708d7ee3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
manage_GSNdata.py
+ 10
− 6
View file @ 52f4f5d7
......@@ -128,8 +128,14 @@ def main():
GSN2IMG = args.gsn2img
N_IMG = args.nImg
KEEP_NEF = False
DOWNLOAD_LIMIT = 10000
# max number of images to download
DOWNLOAD_LIMIT = 30000
#DOWNLOAD_MAX_SIZE = 20000
DOWNLOAD_MAX_SIZE = 1000000
RESIZE = False
#resize_w, resize_h = 356, 536
RESIZE_WIDTH = 712
RESIZE_HEIGHT = 1072
## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ##
## DICTIONARIES ASSIGNING 'VIRTUAL_SENSOR' TO 'POSITION'
......@@ -213,7 +219,7 @@ def main():
if GSN2RAW:
print('Data from position {:s} and virtual sensor {:s} is imported (gsn query) as pd.DataFrame and stored as csv-file.'.format(depo,vsensor))
print('Time interval: ',TBEG, ' to ', TEND)
df = get_GSNdata(deployment, position, vsensor, TBEG, TEND)
df = get_GSNdata(deployment, position, vsensor, TBEG, TEND, download_max_size = DOWNLOAD_MAX_SIZE)
if len(df) > 1:
print('Saving raw data')
if vsensor == 'gps_differential__batch__daily':
......@@ -351,7 +357,7 @@ def main():
# Loop through values for a given key of dictionary DEPO_IMG
for vsensor in list(DEPO_IMG[depo]):
print(deployment, position, vsensor, TBEG, TEND)
df = get_GSNimg(PATH_DATA + '/timelapse_images/', deployment, position, vsensor, TBEG, TEND, n_img = N_IMG, keep_nef = KEEP_NEF, download_limit = DOWNLOAD_LIMIT)
df = get_GSNimg(PATH_DATA + '/timelapse_images/', deployment, position, vsensor, TBEG, TEND, n_img = N_IMG, keep_nef = KEEP_NEF, download_limit = DOWNLOAD_LIMIT, download_max_size = DOWNLOAD_MAX_SIZE, resize = RESIZE, resize_w = RESIZE_WIDTH, resize_h = RESIZE_HEIGHT)
......@@ -389,8 +395,6 @@ def parseArguments():
# Parse argument
return parser.parse_args()
if __name__ == "__main__":
main()
......
permasense/GSNdata.py
+ 50
− 26
View file @ 52f4f5d7
......@@ -130,7 +130,7 @@ def load_data(path_data_file, depo, vsensor, year=[], file_type='csv'):
return df_all
def get_GSNdata(deployment, position, vsensor, tbeg, tend):
def get_GSNdata(deployment, position, vsensor, tbeg, tend, download_max_size):
"""
Get GSNdata with HTTP access
:param deployment: PermaSense deployment (matterhorn, jungfraujoch, dirruhorn, ?)
......@@ -172,9 +172,10 @@ def get_GSNdata(deployment, position, vsensor, tbeg, tend):
# Create query and add time as well as position selection
query= 'vs[1]={:s}&time_format=iso&timeline=generation_time&field[1]=All&from={:s}&to={:s}&c_vs[1]={:s}&c_join[1]=and&c_field[1]=position&c_min[1]={:02d}&c_max[1]={:02d}'.format(virtual_sensor, tbeg.strftime('%d/%m/%Y+%H:%M:%S'), tend.strftime('%d/%m/%Y+%H:%M:%S'), virtual_sensor, int(position)-1, position)
# query extension for images, min/max size for cathcing only NEF images
# query extension for images, min/max size specified for selecting only NEF images or all images
if vsensor == 'binary__mapped':
query = query + '&vs[2]={:s}&field[2]=relative_file&c_join[2]=and&c_vs[2]={:s}&c_field[2]=file_complete&c_min[2]=0&c_max[2]=1&vs[3]={:s}&field[3]=generation_time&c_join[3]=and&c_vs[3]={:s}&c_field[3]=file_size&c_min[3]=2000000&c_max[3]=%2Binf&download_format=csv'.format(virtual_sensor, virtual_sensor, virtual_sensor, virtual_sensor)
#query = query + '&vs[2]={:s}&field[2]=relative_file&c_join[2]=and&c_vs[2]={:s}&c_field[2]=file_complete&c_min[2]=0&c_max[2]=1&vs[3]={:s}&field[3]=generation_time&c_join[3]=and&c_vs[3]={:s}&c_field[3]=file_size&c_min[3]=2000000&c_max[3]=%2Binf&download_format=csv'.format(virtual_sensor, virtual_sensor, virtual_sensor, virtual_sensor)
query = query + '&vs[2]={:s}&field[2]=relative_file&c_join[2]=and&c_vs[2]={:s}&c_field[2]=file_complete&c_min[2]=0&c_max[2]=1&vs[3]={:s}&field[3]=generation_time&c_join[3]=and&c_vs[3]={:s}&c_field[3]=file_size&c_min[3]={:d}&c_max[3]=%2Binf&download_format=csv'.format(virtual_sensor, virtual_sensor, virtual_sensor, virtual_sensor,download_max_size)
# Construct url:
start_time = time.time()
......@@ -380,7 +381,7 @@ def clean_data(df_raw, depo, vsensor):
return df_cleaned
def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_nef, download_limit):
def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_nef, download_limit, download_max_size, resize, resize_w, resize_h):
import os as os
import numpy as np
import pandas as pd
......@@ -388,6 +389,8 @@ def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_
import subprocess, shlex
import piexif
#print(download_limit, download_max_size, resize, resize_w, resize_h)
if isinstance(n_img, int):
pass
elif isinstance(n_img, str) and n_img=='ALL':
......@@ -396,7 +399,8 @@ def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_
print('\"n_img\" not properly defined: \"n_img\" must be an integer value or \"ALL\"')
print('ADD WARNING')
df = get_GSNdata(deployment, position, vsensor, tbeg, tend)
df = get_GSNdata(deployment, position, vsensor, tbeg, tend, download_max_size)
print(df)
df['pk'] = df.data.str.split('=', expand=True)[3]
df['fname'] = df.relative_file.str.split('/', expand=True)[2].str.split('.', expand=True)[0]
df['time'] = df.relative_file.str.split('/', expand=True)[2].str.split('.', expand=True)[0]
......@@ -407,6 +411,10 @@ def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_
count = 0
for index, row in df.iterrows():
count += 1
#skip modulo 2 images
#if count %2 == 0:
# continue
if count > n_img:
print('Downloading {:d} images finished.'.format(n_img))
break
......@@ -422,39 +430,55 @@ def get_GSNimg(path_img, deployment, position, vsensor, tbeg, tend, n_img, keep_
print('Binary file name is not recognized')
continue
# Create subdirectory
#print(row.fname)
#print(row.pk)
subfolder = str(row.fname)[:10]
#print(subfolder)
#subfolder = '{:s}-{:s}-{:s}'.format(row.fname.split('_')[0][0:4], row.fname.split('_')[0][4:6], row.fname.split('_')[0][6:])
os.makedirs(path_img + subfolder, exist_ok=True)
# Path and filename definitions
src = '{:s}{:s}/{:s}.NEF'.format(path_img,subfolder,row.fname.strftime("%Y%m%d_%H%M%S"))
# Query images, convert to .JPG and store
extension = os.path.splitext(row.relative_file)[1]
dst = '{:s}{:s}/{:s}.JPG'.format(path_img,subfolder,row.fname.strftime("%Y%m%d_%H%M%S"))
#print(src)
#print(dst)
# Query images, convert to .JPG and store
urllib.request.urlretrieve('http://data.permasense.ch/field?vs=matterhorn_binary__mapped&field=data&pk={:s}'.format(row.pk), src)
try:
cmd = shlex.split('./NConvert/nconvert -quiet -out jpeg -embedded_jpeg -overwrite -o %s %s' % (dst,src))
if extension in ['.JPG','.jpg']:
# Path and filename definitions
src = '{:s}{:s}/{:s}.jpg'.format(path_img,subfolder,row.fname.strftime("%Y%m%d_%H%M%S"))
urllib.request.urlretrieve('http://data.permasense.ch/field?vs=matterhorn_binary__mapped&field=data&pk={:s}'.format(row.pk), src)
if resize:
cmd = shlex.split('./NConvert/nconvert -quiet -resize %d %d -out jpeg -overwrite -o %s %s' % (resize_h,resize_w,dst,src))
else:
cmd = shlex.split('./NConvert/nconvert -quiet -out jpeg -overwrite -o %s %s' % (dst,src))
proc_util = subprocess.run(cmd, stdout=subprocess.PIPE)
elif extension in ['.NEF','.nef']:
# Path and filename definitions
src = '{:s}{:s}/{:s}.NEF'.format(path_img,subfolder,row.fname.strftime("%Y%m%d_%H%M%S"))
urllib.request.urlretrieve('http://data.permasense.ch/field?vs=matterhorn_binary__mapped&field=data&pk={:s}'.format(row.pk), src)
cmd = shlex.split('./NConvert/nconvert -quiet -embedded_jpeg -rotate 90 -dpi 300 -out jpeg -overwrite -o %s %s' % (dst,src))
proc_util = subprocess.run(cmd, stdout=subprocess.PIPE)
if resize:
cmd = shlex.split('./NConvert/nconvert -quiet -out jpeg -resize %d %d -overwrite -o %s %s' % (resize_h,resize_w,dst,dst))
else:
cmd = shlex.split('./NConvert/nconvert -quiet -out jpeg -overwrite -o %s %s' % (dst,dst))
proc_util = subprocess.run(cmd, stdout=subprocess.PIPE)
exif_dict = piexif.load(dst)
exif_dict["0th"][piexif.ImageIFD.Orientation] = 8
piexif.insert(piexif.dump(exif_dict),dst)
else:
warnings.warn('The file has an unknown extension an will be ignored: %s'% src)
except:
print('Do you have downloaded the NConvert (batch image processor) and copied it into the folder \"gsn_data_management_bin\"?')
pass
proc_util = subprocess.run(cmd, stdout=subprocess.PIPE)
cmd = shlex.split('./NConvert/nconvert -quiet -out jpeg -jpegtrans rot90 -overwrite -o %s %s' % (dst,dst))
proc_util = subprocess.run(cmd, stdout=subprocess.PIPE)
exif_dict = piexif.load(dst)
exif_dict["0th"][piexif.ImageIFD.Orientation] = 8
piexif.insert(piexif.dump(exif_dict),dst)
if ~keep_nef:
pass
# print(src)
# print(dst)
# #print(row.pk)
if not keep_nef:
#print(src, os.path.getsize(src))
os.remove(src)
else:
print(src, os.path.getsize(src))
print(dst, os.path.getsize(dst))
#
#def run_dataavailability_check_MH(tbeg=0,tend=0):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment