diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..d9005f2cc7fc4e65f14ed5518276007c08cf2fd0
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,152 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
diff --git a/pyotb/__init__.py b/pyotb/__init__.py
index 859867a45abca73d79147a5215757e15200dccba..9a5ddf59e73ce59a828c7edb07d11454061171f9 100644
--- a/pyotb/__init__.py
+++ b/pyotb/__init__.py
@@ -1,7 +1,7 @@
-__version__ = "1.0.2"
-
-from .core import App, Output, Input, get_nbchannels, get_pixel_type
+# -*- coding: utf-8 -*-
+__version__ = "1.1"
from .apps import *
-
+from .core import App, Output, Input, get_nbchannels, get_pixel_type
from .functions import *
+from .tools import logger
diff --git a/pyotb/apps.py b/pyotb/apps.py
index 423a5305a3e090030c1229295bd47cf75a44ccb3..c0b7019f00fce3984b6509ab569a50a6dc7ee41d 100644
--- a/pyotb/apps.py
+++ b/pyotb/apps.py
@@ -1,31 +1,88 @@
-import sys
-import subprocess
-from pyotb.core import App, logger
-
+# -*- coding: utf-8 -*-
"""
-This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App('AppName', ...)`
+Search for OTB (set env if necessary), subclass core.App for each available application
"""
+import os
+import sys
+from pathlib import Path
+
+from .tools import logger, find_otb, set_gdal_vars
+
+# Will first call `import otbApplication`, trying to workaround any ImportError
+OTB_ROOT, OTB_APPLICATION_PATH = find_otb()
+
+if not OTB_ROOT:
+ sys.exit("Can't run without OTB. Exiting.")
+
+set_gdal_vars(OTB_ROOT)
+
+# Should not raise ImportError since it was tested in find_otb()
+import otbApplication as otb
+
-AVAILABLE_APPLICATIONS = None
-# Currently there is an incompatibility between OTBTF and Tensorflow that causes segfault when OTB is used in a script
-# where tensorflow has been imported.
-# Thus, we run this piece of code in a clean independent `subprocess` that doesn't interact with Tensorflow
-if sys.executable:
- try:
- p = subprocess.run([sys.executable, '-c', 'import otbApplication; '
- 'print(otbApplication.Registry.GetAvailableApplications())'],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- AVAILABLE_APPLICATIONS = eval(p.stdout.decode().strip())
- except Exception as e:
- logger.warning('Failed to get the list of applications in an independent process. Trying to get it inside'
- 'the script scope')
-
-# In case the previous has failed, we try the "normal" way to get the list of applications
-if not AVAILABLE_APPLICATIONS or not isinstance(AVAILABLE_APPLICATIONS, (list, tuple)):
- import otbApplication
- AVAILABLE_APPLICATIONS = otbApplication.Registry.GetAvailableApplications()
-
-# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App('AppName', ...)`
-if AVAILABLE_APPLICATIONS:
- for app_name in AVAILABLE_APPLICATIONS:
- exec(f"""def {app_name}(*args, **kwargs): return App('{app_name}', *args, **kwargs)""")
+def get_available_applications(as_subprocess=False):
+ """
+ Find available OTB applications
+ :param as_subprocess: indicate if function should list available applications using subprocess call
+ :returns: tuple of available applications
+ """
+ app_list = ()
+ if as_subprocess and sys.executable and hasattr(sys, 'ps1'):
+ # Currently there is an incompatibility between OTBTF and Tensorflow that causes segfault
+ # when OTBTF apps are used in a script where tensorflow has already been imported.
+ # See https://github.com/remicres/otbtf/issues/28
+ # Thus, we run this piece of code in a clean independent `subprocess` that doesn't interact with Tensorflow
+ env = os.environ.copy()
+ if "PYTHONPATH" not in env:
+ env["PYTHONPATH"] = ""
+ env["PYTHONPATH"] = ":" + str(Path(otb.__file__).parent)
+ env["OTB_LOGGER_LEVEL"] = "CRITICAL" # in order to supress warnings while listing applications
+ pycmd = "import otbApplication; print(otbApplication.Registry.GetAvailableApplications())"
+ cmd_args = [sys.executable, "-c", pycmd]
+ try:
+ import subprocess
+ params = {"env": env, "stdout": subprocess.PIPE, "stderr": subprocess.PIPE}
+ with subprocess.Popen(cmd_args, **params) as p:
+ logger.debug(f"{' '.join(cmd_args[:-1])} '{pycmd}'")
+ stdout, stderr = p.communicate()
+ stdout, stderr = stdout.decode(), stderr.decode()
+ # ast.literal_eval is secure and will raise more handy Exceptions than eval
+ from ast import literal_eval
+ app_list = literal_eval(stdout.strip())
+ assert isinstance(app_list, (tuple, list))
+
+ except subprocess.SubprocessError:
+ logger.debug("Failed to call subprocess")
+ except (ValueError, SyntaxError, AssertionError):
+ logger.debug("Failed to decode output or convert to tuple :" + f"\nstdout={stdout}\nstderr={stderr}")
+ if not app_list:
+ logger.info("Failed to list applications in an independent process. Falling back to local otb import")
+
+ if not app_list:
+ app_list = otb.Registry.GetAvailableApplications()
+ if not app_list:
+ logger.warning("Unable to load applications. Set env variable OTB_APPLICATION_PATH then try again")
+ return ()
+
+ logger.info(f"Successfully loaded {len(app_list)} OTB applications")
+ return app_list
+
+
+if OTB_APPLICATION_PATH:
+ otb.Registry.SetApplicationPath(OTB_APPLICATION_PATH)
+ os.environ["OTB_APPLICATION_PATH"] = OTB_APPLICATION_PATH
+
+AVAILABLE_APPLICATIONS = get_available_applications(as_subprocess=True)
+
+# First core.py call (within __init__ scope)
+from .core import App
+
+# This is to enable aliases of Apps, i.e. using apps like `pyotb.AppName(...)` instead of `pyotb.App("AppName", ...)`
+_code_template = """
+class {name}(App):
+ def __init__(self, *args, **kwargs):
+ super().__init__('{name}', *args, **kwargs)
+"""
+# Here we could customize the template and overwrite special methods depending on application
+for _app in AVAILABLE_APPLICATIONS:
+ exec(_code_template.format(name=_app))
diff --git a/pyotb/core.py b/pyotb/core.py
index 19b582b8282a43eda9a4bca8f8b855af8d7b92d7..c8c1ec80a8b8211947cef705b38d55e264d49520 100644
--- a/pyotb/core.py
+++ b/pyotb/core.py
@@ -1,14 +1,12 @@
-import otbApplication
+# -*- coding: utf-8 -*-
import logging
-import sys
-import os
from abc import ABC
+from pathlib import Path
+
import numpy as np
+import otbApplication as otb
-logger = logging
-logger.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
- level=logging.INFO,
- datefmt='%Y-%m-%d %H:%M:%S')
+logger = logging.getLogger()
class otbObject(ABC):
@@ -17,28 +15,6 @@ class otbObject(ABC):
All child of this class must have an `app` attribute that is an OTB application.
"""
-
- def __getitem__(self, key):
- """
- This function enables 2 things :
- - access attributes like that : object['any_attribute']
- - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
- selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
- selecting 1000x1000 subset : object[:1000, :1000]
- """
- # Accessing string attributes
- if isinstance(key, str):
- return self.__dict__.get(key)
-
- # Slicing
- elif not isinstance(key, tuple) or (isinstance(key, tuple) and len(key) < 2):
- raise ValueError('`{}`cannot be interpreted as valid slicing. Slicing should be 2D or 3D.'.format(key))
- elif isinstance(key, tuple) and len(key) == 2:
- # adding a 3rd dimension
- key = key + (slice(None, None, None),)
- (rows, cols, channels) = key
- return Slicer(self, rows, cols, channels)
-
@property
def shape(self):
"""
@@ -48,18 +24,12 @@ class otbObject(ABC):
if hasattr(self, 'output_parameter_key'): # this is for Input, Output, Operation, Slicer
output_parameter_key = self.output_parameter_key
else: # this is for App
- output_parameter_key = self.get_output_parameters_keys()[0]
+ output_parameter_key = self.output_parameters_keys[0]
image_size = self.GetImageSize(output_parameter_key)
image_bands = self.GetImageNbBands(output_parameter_key)
# TODO: it currently returns (width, height, bands), should we use numpy convention (height, width, bands) ?
return (*image_size, image_bands)
- def __getattr__(self, name):
- """This method is called when the default attribute access fails. We choose to try to access the attribute of
- self.app. Thus, any method of otbApplication can be used transparently on otbObject objects,
- e.g. SetParameterOutputImagePixelType() or ExportImage() work"""
- return getattr(self.app, name)
-
def write(self, *args, filename_extension=None, pixel_type=None, is_intermediate=False, **kwargs):
"""
Write the output
@@ -79,30 +49,29 @@ class otbObject(ABC):
"""
# Gather all input arguments in kwargs dict
- if args:
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, str) and kwargs:
- logger.warning('Keyword arguments specified, ignoring argument: {}'.format(arg))
- elif isinstance(arg, str):
- if hasattr(self, 'output_parameter_key'): # this is for Output, Operation
- output_parameter_key = self.output_parameter_key
- else: # this is for App
- output_parameter_key = self.get_output_parameters_keys()[0]
- kwargs.update({output_parameter_key: arg})
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, str) and kwargs:
+ logger.warning(f'{self.name}: Keyword arguments specified, ignoring argument "{arg}"')
+ elif isinstance(arg, str):
+ if hasattr(self, 'output_parameter_key'): # this is for Output, Operation
+ output_parameter_key = self.output_parameter_key
+ else: # this is for App
+ output_parameter_key = self.output_parameters_keys[0]
+ kwargs.update({output_parameter_key: arg})
# Handling pixel types
pixel_types = {}
if isinstance(pixel_type, str): # this correspond to 'uint8' etc...
- pixel_type = getattr(otbApplication, 'ImagePixelType_{}'.format(pixel_type))
+ pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}')
pixel_types = {param_key: pixel_type for param_key in kwargs}
elif isinstance(pixel_type, int): # this corresponds to ImagePixelType_...
pixel_types = {param_key: pixel_type for param_key in kwargs}
elif isinstance(pixel_type, dict): # this is to specify a different pixel type for each output
for key, this_pixel_type in pixel_type.items():
if isinstance(this_pixel_type, str):
- this_pixel_type = getattr(otbApplication, 'ImagePixelType_{}'.format(this_pixel_type))
+ this_pixel_type = getattr(otb, f'ImagePixelType_{this_pixel_type}')
if isinstance(this_pixel_type, int):
pixel_types[key] = this_pixel_type
@@ -117,8 +86,8 @@ class otbObject(ABC):
if not out_fn.endswith('?'):
out_fn += "?"
out_fn += filename_extension
- logger.info("Using extended filename for output.")
- logger.info("Write output for \"{}\" in {}".format(output_parameter_key, out_fn))
+ logger.info(f'{self.name}: Using extended filename for output.')
+ logger.debug(f'{self.name}: write output file "{output_parameter_key}" to {out_fn}')
self.app.SetParameterString(output_parameter_key, out_fn)
if output_parameter_key in pixel_types:
@@ -127,6 +96,36 @@ class otbObject(ABC):
self.app.ExecuteAndWriteOutput()
self.app.Execute() # this is just to be able to use the object in in-memory pipelines without problems
+ # Special methods
+ def __getitem__(self, key):
+ """
+ This function enables 2 things :
+ - access attributes like that : object['any_attribute']
+ - slicing, i.e. selecting ROI/bands. For example, selecting first 3 bands: object[:, :, :3]
+ selecting bands 1, 2 & 5 : object[:, :, [0, 1, 4]]
+ selecting 1000x1000 subset : object[:1000, :1000]
+ """
+ # Accessing string attributes
+ if isinstance(key, str):
+ return self.__dict__.get(key)
+
+ # Slicing
+ elif not isinstance(key, tuple) or (isinstance(key, tuple) and len(key) < 2):
+ raise ValueError('`{}`cannot be interpreted as valid slicing. Slicing should be 2D or 3D.'.format(key))
+ elif isinstance(key, tuple) and len(key) == 2:
+ # adding a 3rd dimension
+ key = key + (slice(None, None, None),)
+ (rows, cols, channels) = key
+ return Slicer(self, rows, cols, channels)
+
+ def __getattr__(self, name):
+ """
+ This method is called when the default attribute access fails. We choose to try to access the attribute of
+ self.app. Thus, any method of otbApplication can be used transparently on otbObject objects,
+ e.g. SetParameterOutputImagePixelType() or ExportImage() work
+ """
+ return getattr(self.app, name)
+
def __add__(self, other):
"""Overrides the default addition and flavours it with BandMathX"""
if isinstance(other, (np.ndarray, np.generic)):
@@ -323,8 +322,7 @@ class Slicer(otbObject):
elif isinstance(channels, tuple):
channels = list(channels)
elif not isinstance(channels, list):
- raise ValueError(
- 'Invalid type for channels, should be int, slice or list of bands. : {}'.format(channels))
+ raise ValueError(f'Invalid type for channels, should be int, slice or list of bands. : {channels}')
# Change the potential negative index values to reverse index
channels = [c if c >= 0 else nb_channels + c for c in channels]
@@ -362,7 +360,6 @@ class Input(otbObject):
"""
Class for transforming a filepath to pyOTB object
"""
-
def __init__(self, filepath):
self.app = App('ExtractROI', filepath).app
self.output_parameter_key = 'out'
@@ -376,7 +373,6 @@ class Output(otbObject):
"""
Class for output of an app
"""
-
def __init__(self, app, output_parameter_key):
self.app = app # keeping a reference of the app
self.output_parameter_key = output_parameter_key
@@ -385,12 +381,34 @@ class Output(otbObject):
return '<pyotb.Output {} object, id {}>'.format(self.app.GetName(), id(self))
+
class App(otbObject):
"""
Class of an OTB app
"""
+ _name = ""
+ @property
+ def name(self):
+ """Property to store a custom app name that will be displayed in logs"""
+ return self._name or self.appname
+
+ @name.setter
+ def name(self, val):
+ self._name = val
- def __init__(self, appname, *args, execute=True, image_dic=None, **kwargs):
+ @property
+ def finished(self):
+ """Property to store whether or not App has been executed"""
+ if self._finished and self.find_output():
+ return True
+ return False
+
+ @finished.setter
+ def finished(self, val):
+ # This will only store if app has been excuted, then find_output() is called when accessing the property
+ self._finished = val
+
+ def __init__(self, appname, *args, execute=True, image_dic=None, otb_stdout=True, **kwargs):
"""
Enables to run an otb app as a oneliner. Handles in-memory connection between apps
:param appname: name of the app, e.g. 'Smoothing'
@@ -407,20 +425,82 @@ class App(otbObject):
:param kwargs: keyword arguments e.g. il=['input1.tif', App_object2, App_object3.out], out='output.tif'
"""
self.appname = appname
- self.app = otbApplication.Registry.CreateApplication(appname)
+ if otb_stdout:
+ self.app = otb.Registry.CreateApplication(appname)
+ else:
+ self.app = otb.Registry.CreateApplicationWithoutLogger(appname)
+ self.image_dic = image_dic
+ self._finished = False
+ # Parameters
+ self.parameters = {}
self.output_parameters_keys = self.get_output_parameters_keys()
- self.set_parameters(*args, **kwargs)
+ if args or kwargs:
+ self.set_parameters(*args, **kwargs)
+ else:
+ logger.warning(f"{self.name}: No parameters where provided. Use App.set_parameters() then App.execute()")
+ execute = False
+ # Run app, write output if needed, update `finished` property
if execute:
- self.app.Execute()
- if image_dic:
- self.image_dic = image_dic
-
+ self.execute()
# 'Saving' outputs as attributes, i.e. so that they can be accessed like that: App.out
# Also, thanks to __getitem__ method, the outputs can be accessed as App["out"]. This is useful when the key
# contains reserved characters such as a point eg "io.out"
- for output_param_key in self.output_parameters_keys:
- output = Output(self.app, output_param_key)
- setattr(self, output_param_key, output)
+ for key in self.output_parameters_keys:
+ output = Output(self.app, key)
+ setattr(self, key, output)
+
+ def execute(self):
+ """
+ Execute with appropriate App method and outputs to disk if needed"
+ :return: boolean flag that indicate if command executed with success
+ """
+ logger.debug(f"{self.name}: run execute() with parameters={self.parameters}")
+ try:
+ if self.__with_output():
+ self.app.ExecuteAndWriteOutput()
+ self.app.Execute()
+ self.finished = True
+ except (RuntimeError, FileNotFoundError) as e:
+ raise Exception(f'{self.name}: error during during app execution') from e
+ logger.debug(f"{self.name}: execution succeeded")
+ return self.finished
+
+ def find_output(self):
+ """
+ Find output files in directory using parameters
+ :return: list of files found on disk
+ """
+ if not self.__with_output():
+ return
+ files = []
+ missing = []
+ for param in self.output_parameters_keys:
+ filename = self.parameters[param]
+ if Path(filename).exists():
+ files.append(filename)
+ else:
+ missing.append(filename)
+ if missing:
+ for filename in missing:
+ logger.error(f"{self.name}: execution seems to have failed, {filename} does not exist")
+ #raise FileNotFoundError(filename)
+ return files
+
+ def clear(self, parameters=True, memory=False):
+ """
+ Free ressources and reset App state
+ :param parameters: to clear settings dictionnary
+ :param memory: to free app ressources in memory
+ """
+ if parameters:
+ for p in self.parameters:
+ self.app.ClearValue(p)
+ if memory:
+ self.app.FreeRessources()
+
+ def get_output_parameters_keys(self):
+ return [param for param in self.app.GetParametersKeys()
+ if self.app.GetParameterType(param) == otb.ParameterType_OutputImage]
def set_parameters(self, *args, **kwargs):
"""
@@ -431,86 +511,100 @@ class App(otbObject):
- string, App or Output, useful when the user implicitly wants to set the param "in"
- list, useful when the user implicitly wants to set the param "il"
:param kwargs: keyword arguments e.g. il=['input1.tif', oApp_object2, App_object3.out], out='output.tif'
- :return:
+ :return: boolean flag that indicate if app was correctly set using given parameters
"""
- if args:
- for arg in args:
- if isinstance(arg, dict):
- kwargs.update(arg)
- elif isinstance(arg, (str, App, Output, Input, Operation)):
- kwargs.update({'in': arg})
- elif isinstance(arg, list):
- kwargs.update({'il': arg})
-
+ self.parameters.update(self.__parse_args(args))
+ self.parameters.update(kwargs)
# Going through all arguments
- for k, v in kwargs.items():
+ for param, obj in self.parameters.items():
+ if param not in self.app.GetParametersKeys():
+ raise Exception(f"{self.name}: parameter '{param}' was not recognized. "
+ f"Available keys are {self.app.GetParametersKeys()}")
# When the parameter expects a list, if needed, change the value to list
- if self.is_key_list(k) and not isinstance(v, (list, tuple)):
- v = [v]
-
- # Single-parameter cases
- if isinstance(v, App):
- self.app.ConnectImage(k, v.app, v.output_parameters_keys[0])
- elif isinstance(v, (Output, Input, Operation)):
- self.app.ConnectImage(k, v.app, v.output_parameter_key)
- elif isinstance(v, otbApplication.Application):
- outparamkey = [param for param in v.GetParametersKeys()
- if v.GetParameterType(param) == otbApplication.ParameterType_OutputImage][0]
- self.app.ConnectImage(k, v, outparamkey)
- elif k == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
- self.app.SetParameterInt('ram', int(v))
- elif not isinstance(v, list): # any other parameters (str, int...)
- self.app.SetParameterValue(k, v)
-
- # Parameter list cases
- else:
- # Images list
- if self.is_key_images_list(k):
- # To enable possible in-memory connections, we go through the list and set the parameters one by one
- for input in v:
- print(input)
- if isinstance(input, App):
- self.app.ConnectImage(k, input.app, input.output_parameters_keys[0])
- elif isinstance(input, (Output, Input, Operation, Slicer)):
- self.app.ConnectImage(k, input.app, input.output_parameter_key)
- elif isinstance(input, otbApplication.Application): # this is for backward comp with plain OTB
- outparamkey = [param for param in input.GetParametersKeys() if
- input.GetParameterType(param) == otbApplication.ParameterType_OutputImage][0]
- self.app.ConnectImage(k, input, outparamkey)
- else: # here `input` should be an image filepath
- # Append `input` to the list, do not overwrite any previously set element of the image list
- self.app.AddParameterStringList(k, input)
-
- # List of any other types (str, int...)
- else:
- self.app.SetParameterValue(k, v)
-
- # Writing outputs to disk if needed
- if any([output_param_key in kwargs for output_param_key in self.output_parameters_keys]):
- self.app.ExecuteAndWriteOutput()
- self.app.Execute() # this is just to be able to use the object in in-memory pipelines without problems
-
- def get_output_parameters_keys(self):
- """
- :return: list of output parameters keys, e.g ['out']
- """
- output_param_keys = [param for param in self.app.GetParametersKeys()
- if self.app.GetParameterType(param) == otbApplication.ParameterType_OutputImage]
- return output_param_keys
-
- def is_key_list(self, key):
- return ((self.app.GetParameterType(key) == otbApplication.ParameterType_InputImageList) or
- (self.app.GetParameterType(key) == otbApplication.ParameterType_StringList) or
- (self.app.GetParameterType(key) == otbApplication.ParameterType_InputFilenameList) or
- (self.app.GetParameterType(key) == otbApplication.ParameterType_InputVectorDataList) or
- (self.app.GetParameterType(key) == otbApplication.ParameterType_ListView))
-
- def is_key_images_list(self, key):
- return ((self.app.GetParameterType(key) == otbApplication.ParameterType_InputImageList) or
- (self.app.GetParameterType(key) == otbApplication.ParameterType_InputFilenameList))
-
+ if self.__is_key_list(param) and not isinstance(obj, (list, tuple)):
+ self.parameters[param] = [obj]
+ obj = [obj]
+ try:
+ # This is when we actually call self.app.SetParameter*
+ self.__set_param(param, obj)
+ except (RuntimeError, TypeError, ValueError) as e:
+ raise Exception(f"{self.name}: something went wrong before execution "
+ f"(while setting parameter {param} to '{obj}')") from e
+
+ # Private functions
+ @staticmethod
+ def __parse_args(args):
+ """Gather all input arguments in kwargs dict"""
+ kwargs = {}
+ for arg in args:
+ if isinstance(arg, dict):
+ kwargs.update(arg)
+ elif isinstance(arg, (str, App, Output, Input, Operation, Slicer)):
+ kwargs.update({'in': arg})
+ elif isinstance(arg, list):
+ kwargs.update({'il': arg})
+ return kwargs
+
+ def __set_param(self, param, obj):
+ """Decide which otb.Application method to use depending on target object"""
+ # Single-parameter cases
+ if isinstance(obj, App):
+ self.app.ConnectImage(param, obj.app, obj.output_parameters_keys[0])
+ elif isinstance(obj, (Output, Input, Operation)):
+ self.app.ConnectImage(param, obj.app, obj.output_parameter_key)
+ elif isinstance(obj, otb.Application): # this is for backward comp with plain OTB
+ outparamkey = [param for param in obj.GetParametersKeys()
+ if obj.GetParameterType(param) == otb.ParameterType_OutputImage][0]
+ self.app.ConnectImage(param, obj, outparamkey)
+ elif param == 'ram': # SetParameterValue in OTB<7.4 doesn't work for ram parameter cf gitlab OTB issue 2200
+ self.app.SetParameterInt('ram', int(obj))
+ elif not isinstance(obj, list): # any other parameters (str, int...)
+ self.app.SetParameterValue(param, obj)
+ ### Parameter list cases
+ # Images list
+ elif self.__is_key_images_list(param):
+ # To enable possible in-memory connections, we go through the list and set the parameters one by one
+ for input in obj:
+ if isinstance(input, App):
+ self.app.ConnectImage(param, input.app, input.output_parameters_keys[0])
+ elif isinstance(input, (Output, Input, Operation, Slicer)):
+ self.app.ConnectImage(param, input.app, input.output_parameter_key)
+ elif isinstance(input, otb.Application): # this is for backward comp with plain OTB
+ outparamkey = [param for param in input.GetParametersKeys() if
+ input.GetParameterType(param) == otb.ParameterType_OutputImage][0]
+ self.app.ConnectImage(param, input, outparamkey)
+ else: # here `input` should be an image filepath
+ # Append `input` to the list, do not overwrite any previously set element of the image list
+ self.app.AddParameterStringList(param, input)
+ # List of any other types (str, int...)
+ else:
+ self.app.SetParameterValue(param, obj)
+
+
+ def __with_output(self):
+ """Check if App has any output parameter key"""
+ return any([k in self.parameters for k in self.output_parameters_keys])
+
+ def __is_key_list(self, key):
+ """Check if a key of the App is an input parameter list"""
+ return self.app.GetParameterType(key) in (
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_StringList,
+ otb.ParameterType_InputFilenameList,
+ otb.ParameterType_InputVectorDataList,
+ otb.ParameterType_ListView
+ )
+
+ def __is_key_images_list(self, key):
+ """Check if a key of the App is an input parameter image list"""
+ return self.app.GetParameterType(key) in (
+ otb.ParameterType_InputImageList,
+ otb.ParameterType_InputFilenameList
+ )
+
+ # Special methods
def __str__(self):
- return '<pyotb.App {} object id {}>'.format(self.appname, id(self))
+ return f'<pyotb.App {self.appname} object id {id(self)}>'
class Operation(otbObject):
@@ -587,7 +681,7 @@ class Operation(otbObject):
self.inputs = []
self.nb_channels = {}
- print(operator, inputs)
+ logger.debug(f"{operator}, {inputs}")
if operator == '?' and nb_bands: # this is when we use the ternary operator with `pyotb.where` function
nb_bands = nb_bands
else:
@@ -704,10 +798,8 @@ class Operation(otbObject):
def __str__(self):
if self.input2 is not None:
- return '<pyotb.Operation object, {} {} {}, id {}>'.format(str(self.input1), self.operator, str(self.input2),
- id(self))
- else:
- return '<pyotb.Operation object, {} {}, id {}>'.format(self.operator, str(self.input1), id(self))
+ return f'<pyotb.Operation object, {self.input1} {self.operator} {self.input2}, id {id(self)}>'
+ return f'<pyotb.Operation object, {self.operator} {self.input1}, id {id(self)}>'
class logicalOperation(Operation):
@@ -718,7 +810,7 @@ class logicalOperation(Operation):
"""
def __init__(self, operator, *inputs, nb_bands=None):
- super().__init__(operator, *inputs)
+ super().__init__(operator, *inputs, nb_bands=nb_bands)
self.logical_exp_bands, self.logical_exp = self.get_real_exp(self.logical_fake_exp_bands)
@@ -766,23 +858,19 @@ def get_nbchannels(inp):
"""
Get the nb of bands of input image
:param inp: a str
+ :return: number of bands in image
"""
if isinstance(inp, otbObject):
nb_channels = inp.shape[-1]
else:
# Executing the app, without printing its log
- stdout = sys.stdout
- sys.stdout = open(os.devnull, 'w')
try:
- info = App("ReadImageInfo", inp)
- sys.stdout = stdout
+ info = App("ReadImageInfo", inp, otb_stdout=False)
nb_channels = info.GetParameterInt("numberbands")
- except Exception as e:
- sys.stdout = stdout
- logger.error('Not a valid image : {}'.format(inp))
- logger.error(e)
+ except (RuntimeError, TypeError) as e:
+ logger.error(f'Not a valid image : {inp}')
+ logger.error(f'{e}')
nb_channels = None
- sys.stdout = stdout
return nb_channels
@@ -795,16 +883,15 @@ def get_pixel_type(inp):
"""
if isinstance(inp, str):
# Executing the app, without printing its log
- stdout = sys.stdout
- sys.stdout = open(os.devnull, 'w')
- info = App("ReadImageInfo", inp)
- sys.stdout = stdout
+ info = App("ReadImageInfo", inp, otb_stdout=False)
datatype = info.GetParameterInt("datatype") # which is such as short, float...
- dataype_to_pixeltype = {'unsigned_char': 'uint8', 'short': 'int16', 'unsigned_short': 'uint16', 'int': 'int32',
- 'unsigned_int': 'uint32', 'long': 'int32', 'ulong': 'uint32', 'float': 'float',
- 'double': 'double'}
+ dataype_to_pixeltype = {
+ 'unsigned_char': 'uint8', 'short': 'int16', 'unsigned_short': 'uint16',
+ 'int': 'int32', 'unsigned_int': 'uint32', 'long': 'int32', 'ulong': 'uint32',
+ 'float': 'float','double': 'double'
+ }
pixel_type = dataype_to_pixeltype[datatype]
- pixel_type = getattr(otbApplication, 'ImagePixelType_{}'.format(pixel_type))
+ pixel_type = getattr(otb, f'ImagePixelType_{pixel_type}')
elif isinstance(inp, (Input, Output, Operation)):
pixel_type = inp.GetImageBasePixelType(inp.output_parameter_key)
elif isinstance(inp, App):
diff --git a/pyotb/functions.py b/pyotb/functions.py
index 435674ab1c1a2b641f8aa8b55ff22326c26fc642..6ad8bf224382d4c504bf372e80686f362435cad6 100644
--- a/pyotb/functions.py
+++ b/pyotb/functions.py
@@ -1,10 +1,12 @@
+# -*- coding: utf-8 -*-
import os
import uuid
+import logging
import multiprocessing
from collections import Counter
-from pyotb.core import (App, Input, Operation, logicalOperation, get_nbchannels, logger)
-
+from .core import (App, Input, Operation, logicalOperation, get_nbchannels)
+logger = logging.getLogger()
"""
Contains several useful functions base on pyotb
"""
@@ -31,8 +33,8 @@ def where(cond, x, y):
if x_nb_channels and y_nb_channels:
if x_nb_channels != y_nb_channels:
- raise Exception('X and Y images do not have the same number of bands. '
- 'X has {} bands whereas Y has {} bands'.format(x_nb_channels, y_nb_channels))
+ raise Exception('X and Y images do not have the same number of bands. ' +
+ f'X has {x_nb_channels} bands whereas Y has {y_nb_channels} bands')
x_or_y_nb_channels = x_nb_channels if x_nb_channels else y_nb_channels
cond_nb_channels = get_nbchannels(cond)
@@ -44,13 +46,13 @@ def where(cond, x, y):
out_nb_channels = cond_nb_channels
if cond_nb_channels != 1 and x_or_y_nb_channels and cond_nb_channels != x_or_y_nb_channels:
- raise Exception('Condition and X&Y do not have the same number of bands. Condition has '
- '{} bands whereas X&Y have {} bands'.format(cond_nb_channels, x_or_y_nb_channels))
+ raise Exception(f'Condition and X&Y do not have the same number of bands. Condition has '
+ '{cond_nb_channels} bands whereas X&Y have {x_or_y_nb_channels} bands')
# If needed, duplicate the single band binary mask to multiband to match the dimensions of x & y
if cond_nb_channels == 1 and x_or_y_nb_channels and x_or_y_nb_channels != 1:
- logger.info('The condition has one channel whereas X/Y has/have {} channels. Expanding number of channels '
- 'of condition to match the number of channels or X/Y'.format(x_or_y_nb_channels))
+ logger.info(f'The condition has one channel whereas X/Y has/have {x_or_y_nb_channels} channels. Expanding number of channels '
+ 'of condition to match the number of channels or X/Y')
operation = Operation('?', cond, x, y, nb_bands=out_nb_channels)
@@ -233,8 +235,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
# TODO : it is when the user wants the final bounding box to be the union of all bounding box
# It should replace any 'outside' pixel by some NoData -> add `fillvalue` argument in the function
- logger.info(
- 'Cropping all images to extent Upper Left ({}, {}), Lower Right ({}, {}) '.format(ULX, ULY, LRX, LRY))
+ logger.info(f'Cropping all images to extent Upper Left ({ULX}, {ULY}), Lower Right ({LRX}, {LRY})')
# Applying this bounding box to all inputs
new_inputs = []
@@ -251,7 +252,7 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
reference_pixel_size_input = new_input
except Exception as e:
logger.error(e)
- logger.error('Images may not intersect : {}'.format(input))
+ logger.error(f'Images may not intersect : {input}')
# TODO: what should we do then? return an empty raster ? fail ? return None ?
inputs = new_inputs
@@ -277,13 +278,13 @@ def define_processing_area(*args, window_rule='intersection', pixel_size_rule='m
elif pixel_size_rule == 'specify':
pass
# TODO : when the user explicitely specify the pixel size -> add argument inside the function
-
- logger.info('Resampling all inputs to resolution : {}'.format(metadatas[reference_input]['GeoTransform'][1]))
+ pixel_size = metadatas[reference_input]['GeoTransform'][1]
+ logger.info(f'Resampling all inputs to resolution : {pixel_size}')
# Perform resampling on inputs that do not comply with the target pixel size
new_inputs = []
for input in inputs:
- if metadatas[input]['GeoTransform'][1] != metadatas[reference_input]['GeoTransform'][1]:
+ if metadatas[input]['GeoTransform'][1] != pixel_size:
superimposed = App('Superimpose', inr=reference_input, inm=input, interpolator=interpolator)
new_inputs.append(superimposed)
else:
@@ -378,7 +379,7 @@ def run_tf_function(func):
# Create and save the model. This is executed **inside an independent process** because (as of 2021-11),
# tensorflow python library and OTBTF are incompatible
- out_savedmodel = os.path.join(tmp_dir, 'tmp_otbtf_model_{}'.format(uuid.uuid4()))
+ out_savedmodel = os.path.join(tmp_dir, f'tmp_otbtf_model_{uuid.uuid4()}')
p = multiprocessing.Process(target=create_and_save_tf_model, args=(out_savedmodel, *inputs,))
p.start()
p.join()
@@ -394,7 +395,7 @@ def run_tf_function(func):
'optim.disabletiling': 'on', 'model.fullyconv': 'on'}, execute=False)
for i, input in enumerate(raster_inputs):
- model_serve.set_parameters({'source{}.il'.format(i + 1): [input]})
+ model_serve.set_parameters({f'source{i + 1}.il': [input]})
model_serve.Execute()
# TODO: handle the deletion of the temporary model ?
diff --git a/pyotb/tools.py b/pyotb/tools.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e82f850a8671c57fec769aa932d79a1ca8bb0c6
--- /dev/null
+++ b/pyotb/tools.py
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+import os
+import sys
+import logging
+from pathlib import Path
+
+# This will prevent erasing user config
+if "logger" not in globals():
+ logging.basicConfig(
+ format="%(asctime)s (%(levelname)-4s) [pyOTB] %(message)s",
+ level=logging.INFO,
+ datefmt="%Y-%m-%d %H:%M:%S",
+ )
+logger = logging.getLogger()
+
+
+def find_otb(root="", scan=True, scan_userdir=True):
+ """
+ Try to load OTB bindings or scan system, help user in case of failure, return env variables
+ Precedence : OTB_ROOT > python bindings directory
+ OR search for releases installations : current directory > home directory
+ OR (for linux) : /opt/otbtf > /opt/otb > /usr/local > /usr
+ :param root: prefix to search OTB in
+ :param scan: find otb in system known locations
+ :param scan_userdir: search for OTB release in user's home directory
+ :return: (OTB_ROOT, OTB_APPLICATIONS_PATH) environment variables
+ """
+
+ # OTB_ROOT env variable is first (allow override default OTB version)
+ prefix = root or os.environ.get("OTB_ROOT") or os.environ.get("OTB_DIR")
+ if prefix:
+ logger.info(f"Found OTB Python API in {prefix}")
+ # Add path first in PYTHONPATH
+ otb_api = get_python_api(prefix)
+ sys.path.insert(0, str(otb_api))
+ if "otb" not in globals():
+ try:
+ import otbApplication
+ lib_dir = get_lib(otbApplication)
+ apps_path = get_apps_path(lib_dir)
+ prefix = lib_dir.parent
+ if prefix.name == 'lib':
+ prefix = prefix.parent
+ return str(prefix), str(apps_path)
+
+ except ImportError:
+ logging.critical(f"Can't import OTB with OTB_ROOT={prefix}")
+ return None, None
+ else:
+ logger.warning("Can't reset otb module with new path. You need to restart Python")
+
+ # Else try with python file location
+ try:
+ import otbApplication
+ lib_dir = get_lib(otbApplication)
+ apps_path = get_apps_path(lib_dir)
+ logger.debug("Found OTB directory using Python lib location")
+ return str(lib_dir.parent), str(apps_path)
+
+ # Else search system
+ except ImportError:
+ PYTHONPATH = os.environ.get("PYTHONPATH")
+ logger.info(f"Failed to import otbApplication with PYTHONPATH={PYTHONPATH}")
+ if not scan:
+ return None, None
+ logger.info("Searching for it...")
+ lib_dir = None
+ # Scan user's HOME directory tree (this may take some time)
+ if scan_userdir :
+ for path in Path().home().glob("**/OTB-*/lib"):
+ logger.info(f"Found {path.parent}")
+ lib_dir = path
+ prefix = str(path.parent.absolute())
+ # Or search possible known locations (system scpecific)
+ if sys.platform == "linux":
+ possible_locations = (
+ "/usr/lib/x86_64-linux-gnu/otb", "/usr/local/lib/otb/",
+ "/opt/otb/lib/otb/", "/opt/otbtf/lib/otb",
+ )
+ for str_path in possible_locations:
+ path = Path(str_path)
+ if not path.exists():
+ continue
+ logger.info(f"Found " + str_path)
+ if not prefix:
+ if path.parent.name == "x86_64-linux-gnu":
+ prefix = path.parent.parent.parent
+ else:
+ prefix = path.parent.parent
+ prefix = str(prefix.absolute())
+ lib_dir = path.parent.absolute()
+ else:
+ # TODO: find OTB path in other OS ? pathlib should help
+ pass
+
+ # Found OTB
+ if prefix and lib_dir is not None:
+ otb_api = get_python_api(prefix)
+ if otb_api is None or not otb_api.exists():
+ logger.error("Can't find OTB python API")
+ return None, None
+
+ # Try to import one last time before sys.exit (in apps.py)
+ try:
+ sys.path.insert(0, str(otb_api))
+ import otbApplication as otb
+ logger.info(f"Using OTB in {prefix}")
+ return prefix, get_apps_path(lib_dir)
+ except ModuleNotFoundError:
+ logger.critical(f"Unable to find OTB Python bindings", exc_info=1)
+ return None, None
+ except ImportError as e:
+ logger.critical(f"An error occured while importing Python API")
+ if str(e).startswith('libpython3.'):
+ logger.critical("It seems like you need to symlink or recompile OTB SWIG bindings")
+ if sys.platform == "linux":
+ logger.critical(f"Use 'cd {prefix} ; source otbenv.profile ; ctest -S share/otb/swig/build_wrapping.cmake -VV'")
+ return None, None
+ logger.critical("full traceback", exc_info=1)
+
+ return None, None
+
+
+def get_python_api(prefix):
+ root = Path(prefix)
+ if root.exists():
+ otb_api = root / "lib/python"
+ if not otb_api.exists():
+ otb_api = root / "lib/otb/python"
+ if not otb_api.exists():
+ return
+ return otb_api.absolute()
+
+
+def get_lib(otb_module):
+ lib_dir = Path(otb_module.__file__).parent.parent
+ # OTB .run file
+ if lib_dir.name == "lib":
+ return lib_dir.absolute()
+ # Case /usr
+ lib_dir = lib_dir.parent
+ if lib_dir.name in ("lib", "x86_64-linux-gnu"):
+ return lib_dir.absolute()
+ # Case built from source (/usr/local, /opt/otb, ...)
+ lib_dir = lib_dir.parent
+ return lib_dir.absolute()
+
+
+def get_apps_path(lib_dir):
+ if isinstance(lib_dir, Path) and lib_dir.exists():
+ logger.debug(f"Using library from {lib_dir}")
+ otb_application_path = lib_dir / "otb/applications"
+ if otb_application_path.exists():
+ return str(otb_application_path.absolute())
+ # This should not happen, may be with failed builds ?
+ logger.debug(f"Library directory found but no 'applications' directory whithin it")
+ return ""
+
+
+def set_gdal_vars(root):
+ if (Path(root) / "share/gdal").exists():
+ # Local GDAL (OTB Superbuild, .run, .exe)
+ gdal_data = str(Path(root + "/share/gdal"))
+ proj_lib = str(Path(root + "/share/proj"))
+ elif sys.platform == "linux":
+ # If installed using apt or built from source with system deps
+ gdal_data = "/usr/share/gdal"
+ proj_lib = "/usr/share/proj"
+ else:
+ logger.warning(f"Can't find GDAL directory with prefix {root}")
+ return False
+ # Not sure if SWIG will see these
+ os.environ["LC_NUMERIC"] = "C"
+ os.environ["GDAL_DATA"] = gdal_data
+ os.environ["PROJ_LIB"] = proj_lib
+ return True
diff --git a/pyproject.toml b/pyproject.toml
index 374b58cbf4636f1e28bacf987ac2fe89ed27ccba..ddb1465f3a361addd19954bd24367980b66ec754 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,7 @@
[build-system]
requires = [
"setuptools>=42",
- "wheel"
+ "wheel",
+ "numpy>=1.13,<1.23"
]
build-backend = "setuptools.build_meta"
diff --git a/scripts/replace_fstrings.py b/scripts/replace_fstrings.py
new file mode 100755
index 0000000000000000000000000000000000000000..b3568507f5740b5d67989f70c36691fa5308a309
--- /dev/null
+++ b/scripts/replace_fstrings.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import re
+import sys
+
+assert sys.version_info.minor >= 6
+
+
+def replace_fstrings(py_file, out_file):
+ """
+ Replace fstrings with .format() method
+ :param py_file: path for input python file to convert
+ :param out_file: converted file output path
+ """
+
+ def replace(line):
+ exp = re.compile(r"""(.*[\s\(\[{=])f(['"])(.*)(['"])(.*)\n""")
+ res = re.match(exp, line)
+ if not res:
+ return line
+ prefix, qt1, string, qt2, suffix = res.groups()
+ args = []
+ for part in string.split('{')[1:]:
+ a = part.split('}')[0]
+ string = string.replace(a, '')
+ args.append(a)
+
+ return f"""{prefix}{qt1}{string}{qt2}.format({', '.join(args)}){suffix}\n"""
+
+ with open(py_file, 'r') as r:
+ lines = list(map(replace, r.readlines()))
+ with open(out_file, 'w') as w:
+ for li in lines:
+ w.write(li)
+
+if __name__ == "__main__":
+ replace_fstrings(sys.argv[1], sys.argv[2])
diff --git a/setup.py b/setup.py
index f320aaff32e3a14c04a6642189d23451e81f81b2..89e949576a620bd0307a8237373571db0566ac03 100644
--- a/setup.py
+++ b/setup.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import setuptools
with open("README.md", "r", encoding="utf-8") as fh:
@@ -5,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="pyotb",
- version="1.0.2",
+ version="1.1",
author="Nicolas Narçon",
author_email="nicolas.narcon@gmail.com",
description="Library to enable easy use of the Orfeo Tool Box (OTB) in Python",